[ Jun 17 21:16:25 Enabled. ] [ Jun 17 21:16:25 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jun 17 21:16:25 Method "start" exited with status 0. ] [ Jun 17 21:16:25 Rereading configuration. ] [ Jun 17 21:16:25 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:25.959995901Z","hostname":"sock","pid":1555,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:25.960173463Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.014433552Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.014475723Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.129370748Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.138549341Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.146825165Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.751747971Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.893725372Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.893765072Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.893832062Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.893903383Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.893916074Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.893932874Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.897118186Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:26.897127186Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.897140656Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.897145686Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.897159626Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.897201437Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.897275277Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.897290848Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.897308528Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.908949986Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:26.908983976Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.008187732Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.215128511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.215166182Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.215172462Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.215178112Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.215192352Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.215197452Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.215215742Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.223966211Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.223991281Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.31257061Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521176846Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521218956Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521246466Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521260577Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521266337Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521271077Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521282197Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521312867Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521322867Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521336197Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521341747Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521346818Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521350678Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521358458Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521381008Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521385758Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521390648Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521396358Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521400128Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521404198Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521423028Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521428308Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521434078Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521438068Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521442659Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521447539Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521472049Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521477789Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521482599Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521486419Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521491109Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521496669Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521501089Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521505829Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521524209Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521530119Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521536639Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521541749Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.52154616Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.52157214Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.52157836Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.5215835Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.52158769Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.52159533Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.52160029Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.5216188Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.52162734Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.52163226Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.52163611Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.52164095Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521652381Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521657141Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521677781Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521683021Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521686991Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521701321Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521706851Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521743331Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521748651Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521752722Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521756742Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521764302Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521780712Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521789082Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521806742Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521811912Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521816422Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521834672Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.521842183Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.533100557Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.533135747Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99994512s","next_cleanup":"Instant { tv_sec: 1309, tv_nsec: 281008351 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.533165137Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3647"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.533212338Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.533264528Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.533270888Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536280569Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536345699Z","hostname":"sock","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536569702Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1011"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536575462Z","hostname":"sock","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536621962Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536666553Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536681353Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536687943Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1071"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536715383Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536735203Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536744994Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536761774Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1112"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536765344Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536768754Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536776054Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.536801274Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537143128Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537293509Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537301259Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1145"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.53734036Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.53734596Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537489211Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537619443Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537657553Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 09996097-89b8-4c39-a1ef-fd8405b108d6, rack_id: 8e338aae-2fb5-450b-bed3-73c865b83f20, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537786614Z","hostname":"sock","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:54090","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 09996097-89b8-4c39-a1ef-fd8405b108d6, rack_id: 8e338aae-2fb5-450b-bed3-73c865b83f20, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537884185Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537898365Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537920906Z","hostname":"sock","pid":1555,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537928036Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537931726Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537935766Z","hostname":"sock","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537942596Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537953926Z","hostname":"sock","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537987066Z","hostname":"sock","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.537997366Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.538003887Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.540503512Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3879"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.548053238Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.557610445Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.568635827Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.578541798Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.587446728Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.596902614Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.605204948Z","hostname":"sock","pid":1555,"sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:354"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.605234648Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.605924215Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Failed enable ddm stats (will retry after 84.496216ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.605970006Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f69d9ca6-59e7-4ef9-affb-e53571a763df\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:16:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f69d9ca6-59e7-4ef9-affb-e53571a763df\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.614745425Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.623182191Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.623212881Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.623218861Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.623227291Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.623233461Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.623238251Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.623262451Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.623303642Z","hostname":"sock","pid":1555,"sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:366"} {"msg":"attempting to create encrypted zvol: name=\"oxi_506b1cae-3bee-435f-ba3e-326026eb5f18/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.635901209Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.64782493Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"successfully created encrypted zvol: name=\"oxi_506b1cae-3bee-435f-ba3e-326026eb5f18/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.658675351Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_506b1cae-3bee-435f-ba3e-326026eb5f18/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.658716711Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.663689141Z","hostname":"sock","pid":1555,"sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:381"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.663698031Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_506b1cae-3bee-435f-ba3e-326026eb5f18/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.663703642Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.666078906Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.690851187Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 106.825848ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.693623055Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"379e5226-9bb0-400f-bdcd-5dc3c908a7aa\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:16:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"379e5226-9bb0-400f-bdcd-5dc3c908a7aa\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.735927734Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.801106825Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 170.042995ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.801685891Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"035fb2d2-6183-416d-b26e-5fd7d7be1af1\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:16:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"035fb2d2-6183-416d-b26e-5fd7d7be1af1\" }"} {"msg":"Mounting oxi_506b1cae-3bee-435f-ba3e-326026eb5f18/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.954244529Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.967325701Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.967407012Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.967423982Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.967478563Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.972852797Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 97.005573ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:27.973421072Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dff8e917-80b7-41a2-b2c7-4aaaf0d94479\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:16:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dff8e917-80b7-41a2-b2c7-4aaaf0d94479\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.070636698Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 138.373954ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.071058393Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e93f7800-e47c-4aa4-b762-3b0699296204\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:16:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e93f7800-e47c-4aa4-b762-3b0699296204\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.210200494Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 331.170895ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.211032113Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"81aaff76-d9f3-4e91-8b99-ae40063d8bf2\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:16:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"81aaff76-d9f3-4e91-8b99-ae40063d8bf2\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.543181192Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 331.288411ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.543896729Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"465efb79-82eb-4410-a806-eccc5ba9e431\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:16:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"465efb79-82eb-4410-a806-eccc5ba9e431\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.670589824Z","hostname":"sock","pid":1555,"sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.67117806Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:40617","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.67120804Z","hostname":"sock","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.671277071Z","hostname":"sock","pid":1555,"file":"oximeter/producer/src/lib.rs:276","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:276","interval":"30s","address":"[fd00:1122:3344:101::1]:40617","producer_id":"09996097-89b8-4c39-a1ef-fd8405b108d6"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:28.677669736Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"145.605457ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.718213117Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.718246327Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.718388349Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.718400009Z","hostname":"sock","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.719259428Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.719285648Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.719323458Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.719337549Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.719613381Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.719807703Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.719850954Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.719867534Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3647"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.719871954Z","hostname":"sock","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:54090","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.719939365Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.719946385Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.886619365Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.886693766Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"No ledger in /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.886737197Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.886747737Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.886755367Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.886789067Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:28.887854638Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"448.986538ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:28.89199483Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.918531939Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48442"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.918695361Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.918767912Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"a602a5aa-193b-4954-8cac-cf27d57f6be7","remote_addr":"[fd00:1122:3344:101::1]:48442","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.952839927Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:54364"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.952970539Z","hostname":"sock","pid":1555,"uri":"/sled-role","method":"GET","req_id":"e09d95a4-1b09-4f42-b24b-6407fb74863b","remote_addr":"[fd00:1122:3344:101::1]:54364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":73,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.953816817Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.953832547Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.954358073Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.954749997Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:812"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 9175b8a1-7790-4929-8bc6-64201a678d04, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool) }, OmicronPhysicalDiskConfig { id: 1a8b1c6f-4f53-4a0d-b0f1-abf0cfdde603, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool) }, OmicronPhysicalDiskConfig { id: 35b7216c-4b6b-4e8f-9b71-f27570153d53, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool) }, OmicronPhysicalDiskConfig { id: b6b8eeb8-f814-49ac-8ae5-008152c31ce7, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool) }, OmicronPhysicalDiskConfig { id: 0e5f8130-0009-4cb1-a798-bee2295f07a3, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool) }, OmicronPhysicalDiskConfig { id: 14a46aef-e05f-4b56-8f5c-04faaf9d67d1, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool) }, OmicronPhysicalDiskConfig { id: 03f6e1d9-07c5-498d-a840-fc5bf9fe2fbb, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool) }, OmicronPhysicalDiskConfig { id: 3da99a96-cf6a-4186-b570-623b2e6e6cfc, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool) }, OmicronPhysicalDiskConfig { id: fd6102be-a4fe-42d1-9ea4-712b9c1932c9, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.987966183Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:348"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.988129835Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55080"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 9175b8a1-7790-4929-8bc6-64201a678d04, pool_id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 1a8b1c6f-4f53-4a0d-b0f1-abf0cfdde603, pool_id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 35b7216c-4b6b-4e8f-9b71-f27570153d53, pool_id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: b6b8eeb8-f814-49ac-8ae5-008152c31ce7, pool_id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 0e5f8130-0009-4cb1-a798-bee2295f07a3, pool_id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 14a46aef-e05f-4b56-8f5c-04faaf9d67d1, pool_id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 03f6e1d9-07c5-498d-a840-fc5bf9fe2fbb, pool_id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 3da99a96-cf6a-4186-b570-623b2e6e6cfc, pool_id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: fd6102be-a4fe-42d1-9ea4-712b9c1932c9, pool_id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.988370777Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.988385057Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.988390807Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.988395398Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.988405187Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.988409918Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.988415568Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:28.988419808Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:29.091386452Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:29.091428343Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:29.441690656Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.121527119s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-17T21:16:29.878087942Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:30.064044878Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:30.227508456Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:30.227640067Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:30.227651327Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:30.227675477Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:30.227683137Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:30.227847319Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:30.227855849Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:30.582021292Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.573730319s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:30.756282399Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:30.757662263Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:30.94261606Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Automatically destroying dataset: oxp_ffe46556-d7d9-435b-85f0-4376aa51f547/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:31.127695136Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-17T21:16:31.577074815Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 52.495327ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:31.751982909Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 99.304622ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:32.102594435Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 240.386683ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:32.474441257Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 450.371054ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:33.068066658Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:33.071049928Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-17T21:16:33.071070438Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:33.234775578Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.72113381s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 826.853783ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:33.874438637Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-17T21:16:34.117879726Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_ffe46556-d7d9-435b-85f0-4376aa51f547","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:34.12910416Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:34.335332552Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:34.335403652Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:34.335416822Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:34.335427072Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:34.335433782Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:34.561035631Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:34.561060711Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:35.710398959Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:35.949938749Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:35.94999395Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:35.950006599Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:35.95005392Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:35.95006987Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:35.950160391Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:35.950171201Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:36.701380221Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-17T21:16:36.701420981Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:37.425767658Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_2abfdf07-7cd1-4355-89f0-e7d8ec686d7c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:37.916118462Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-17T21:16:38.443223638Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:38.443254688Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:38.668051608Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2333"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:38.895213363Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:38.958404434Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.278726001s"} {"msg":"Looking for unencrypted datasets in oxp_2abfdf07-7cd1-4355-89f0-e7d8ec686d7c","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.007577452Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.015637734Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.015695974Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.015707894Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.015717205Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.015725845Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.026830347Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.026859348Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"e4f51e96-8b35-4542-822a-22c6778e005f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.117286315Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.125963323Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.126000393Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.126007903Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.126027644Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.126035914Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.126158865Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.126170315Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.159729455Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e4f51e96-8b35-4542-822a-22c6778e005f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.167135961Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_e4f51e96-8b35-4542-822a-22c6778e005f","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.282538941Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.289537482Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.289589593Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.289602003Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.289608273Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.289624443Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.299432452Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.299471653Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.394937831Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.402848401Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.402900962Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.402913812Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.402942113Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.402958313Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.403139735Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.403153475Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.437796396Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-17T21:16:39.44509832Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Automatically destroying dataset: oxp_d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.445791087Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.566752664Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.574633794Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.574705755Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.574715825Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.574725325Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.574733925Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.584357713Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.584385953Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"38ec74ae-3290-4738-828b-35608cdf0b99 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.674668109Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.68367005Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.6837047Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.683761361Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.683797091Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.683810842Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.684005223Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.684020033Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.724567775Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_38ec74ae-3290-4738-828b-35608cdf0b99/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.733057001Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_38ec74ae-3290-4738-828b-35608cdf0b99","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.8542151Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.862276272Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.862333862Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.862344192Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.862351193Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.862357393Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.871758508Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.871791008Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.963037564Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.970713711Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.970742081Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.970749451Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.970780642Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.970789912Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.970948863Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:39.970962643Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.001968248Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_cea92678-6b55-4860-a3b6-57f49d0a71a4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.01006812Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_cea92678-6b55-4860-a3b6-57f49d0a71a4","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.1283262Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.135341891Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.135378611Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.135393111Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.135398731Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.135403302Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.14510539Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.14513299Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.240888341Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.248819192Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.248857212Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.248866022Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.248895903Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.248907523Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.249073415Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.249102365Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:40.253085765Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.283277911Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ab647d9b-1f8b-4c75-927b-5b8bc52c8141/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.291302423Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_ab647d9b-1f8b-4c75-927b-5b8bc52c8141","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.413948617Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.421640665Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.421693526Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.421711476Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.421721206Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.421732276Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.431834958Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.431864439Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.528736021Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.536624091Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.536668612Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.536678202Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.536733052Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.536748302Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.536922284Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.536934364Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-17T21:16:40.540978095Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.575258963Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_fa78ab30-cb65-4bf2-8677-79fb8f81cd6b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.585116953Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_fa78ab30-cb65-4bf2-8677-79fb8f81cd6b","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.713165672Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.721019482Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.721099583Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.721108783Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.721116633Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.721123513Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.730953092Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.730983233Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.825830695Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.833122299Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.833164429Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.833173479Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.833180079Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.833189509Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.833326981Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.833349361Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.868315446Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_708ddef6-4ed2-4ac7-bfb0-97e7a7008313/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:40.87756477Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_708ddef6-4ed2-4ac7-bfb0-97e7a7008313","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.001401955Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.009304885Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.009347036Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.009405826Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.009543038Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 506b1cae-3bee-435f-ba3e-326026eb5f18 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: b8f4ab32-a0e3-4946-a6a4-a7402b5b0422 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.009591598Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.009598828Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Writing ledger to /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.00973329Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.010077583Z","hostname":"sock","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"4cafad2f-fdf1-4dc9-8abd-415690941c24","remote_addr":"[fd00:1122:3344:101::1]:55080","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":12021868,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.040510482Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.040649643Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:43667"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.040876866Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.040881846Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.092132766Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.092174176Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.140296654Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.140341315Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.19112018Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.19116148Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.19116989Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.19119796Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.236130276Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.236170687Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.236179417Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.290391817Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.294505058Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.29860678Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.311177277Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.315634703Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:41.320244079Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-17T21:16:41.542961759Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/09996097-89b8-4c39-a1ef-fd8405b108d6): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:42.250781038Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Profile for oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:42.850469931Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:42.850570502Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:42.850801684Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:43.335366929Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:43.33541858Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:43.814476079Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43\") failed: Property not found. retry in 57.040542ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:43.821613931Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:43.821678232Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:43.825727263Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:43.825808114Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43\") failed: Property not found. retry in 127.446216ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:43.889236347Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2\") failed: Property not found. retry in 33.463828ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:43.952811332Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76\") failed: Property not found. retry in 66.012309ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:43.959098506Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2\") failed: Property not found. retry in 78.632686ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:43.994607386Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43\") failed: Property not found. retry in 211.321766ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:44.111076087Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76\") failed: Property not found. retry in 123.879992ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:44.290374746Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2\") failed: Property not found. retry in 277.683612ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:44.314543311Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43\") failed: Property not found. retry in 535.654713ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:44.334725725Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76\") failed: Property not found. retry in 201.805471ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:44.443208876Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2\") failed: Property not found. retry in 247.6545ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:44.604210199Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76\") failed: Property not found. retry in 426.136371ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:44.656194966Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2\") failed: Property not found. retry in 1.176459877s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:44.861838122Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43\") failed: Property not found. retry in 618.740791ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:44.878582302Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76\") failed: Property not found. retry in 1.183627499s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:45.091180358Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43\") failed: Property not found. retry in 990.969292ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:45.50935558Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2\") failed: Property not found. retry in 1.138425748s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:46.046475738Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76\") failed: Property not found. retry in 991.756786ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:46.283439951Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43\") failed: Property not found. retry in 604.57427ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:46.507528794Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:46.59588994Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2350"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:46.595901351Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43\") failed: Property not found. retry in 1.041746299s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:47.120622233Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe73:1840","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:47.142180701Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2357"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:47.148156622Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2418"} {"msg":"wait for service Some(\"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2\") failed: Property not found. retry in 954.65932ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:47.193106918Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:47.270115339Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2603"} {"msg":"wait for service Some(\"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76\") failed: Property not found. retry in 1.25675343s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:47.283664616Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:47.519802442Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2767"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:47.712424805Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2482"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:48.029198298Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2517"} {"msg":"wait for service Some(\"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2\") failed: Property not found. retry in 814.863133ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:48.157194556Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43\") failed: Property not found. retry in 1.196937909s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:48.172651553Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:48.242800685Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2817"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:48.42865945Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2844"} {"msg":"wait for service Some(\"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76\") failed: Property not found. retry in 771.876174ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:48.549387995Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:48.664763855Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2711"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:48.863527941Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2511"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:48.87424931Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3903","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:48.87429731Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3919"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:48.874302921Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.387824689Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_internal_dns_05af6327-e353-49c1-8e4c-eb7263dada76"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.38788765Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_internal_dns_91162038-95e7-4445-b7b9-254e840c7a43"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.38789563Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_internal_dns_278c789c-97ef-4df7-ba73-58ef0dc8f8c2"} {"msg":"Writing ledger to /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.38790078Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.388270514Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.388543776Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"340e83b7-6545-407a-9284-7beb1daf57f1","remote_addr":"[fd00:1122:3344:101::1]:43667","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8347819,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.390194723Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.433845056Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3927"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.440041069Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3983"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.444831457Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.578245911Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4045"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.671433426Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4026","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.693265737Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4073"} {"msg":"failed to write DNS configuration (will retry in 107.934645ms)","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:49.707554022Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:569","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:2::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.792023729Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4100"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.833376238Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.897727531Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:585"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.897770592Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.914433531Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:365"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.952012482Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.953482787Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.953490087Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:49.976907184Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:413"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.011443554Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.011653686Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52564"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.012303833Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.012317653Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.033537108Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.033572099Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.037526389Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:651"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.03762705Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:689"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.038128915Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:692","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.038152595Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:435","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.051895214Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.051938765Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.069265941Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.069314061Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.069468973Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.069599744Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.081400494Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.129231539Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.131901856Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.131912576Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132866526Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132898696Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132904356Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132909006Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132913576Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132929056Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132932876Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132937377Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132940897Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132945177Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132949287Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132952837Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132956187Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132959517Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132962827Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132966147Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132969367Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132976027Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132979437Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132982687Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132985997Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132989257Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.132992557Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.176523489Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 133, 78]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.176573389Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.177251576Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.177259896Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.17759437Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.18058687Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.205406152Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.223557466Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.852678627Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:50.852766638Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\") failed: Property not found. retry in 31.239312ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:50.98118947Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\") failed: Property not found. retry in 60.587088ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:51.021029684Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\") failed: Property not found. retry in 244.455498ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:51.094327698Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\") failed: Property not found. retry in 312.396097ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:51.350028732Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\") failed: Property not found. retry in 410.961208ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:51.670964617Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\") failed: Property not found. retry in 1.384416864s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:52.091091118Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:53.316089093Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50764"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:53.316198934Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"c4027e87-6b3c-4328-95d8-900fe5fcef4c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50764","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\") failed: Property not found. retry in 1.260364161s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:53.484273969Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\") failed: Property not found. retry in 585.211929ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:54.75518167Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\") failed: Property not found. retry in 1.358383691s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:55.349825562Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:56.727284723Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4"} {"msg":"Writing ledger to /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:56.727369684Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:56.727826509Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:56.728186952Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"803d2fc1-e597-474f-a71b-d74e8009d2f6","remote_addr":"[fd00:1122:3344:101::1]:52564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":6716232,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:56.730020731Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:626"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:56.758727062Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:56.758899164Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:53940"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:56.767889095Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3633"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:56.768064397Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"d6eadea5-7741-4428-aed4-80d633949da6","remote_addr":"[fd00:1122:3344:101::1]:53940","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9003,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:16:56.770486881Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:56.879410816Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:16:56.879573258Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:38785"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:00.888814164Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3633"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:00.888912115Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"48ca4387-a847-4bef-8464-09034b375e2a","remote_addr":"[fd00:1122:3344:101::1]:38785","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009202,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:00.892320129Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:01.060167242Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:01.060277623Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:65071"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:01.068986561Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3633"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:01.069124492Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"33efe9e8-37ab-4057-a782-5ba580acc886","remote_addr":"[fd00:1122:3344:101::1]:65071","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8705,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:01.070950551Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:01.304434359Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:01.304563921Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:58522"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:01.313291499Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3633"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:01.31341952Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"2aceea82-d7e9-4007-93f6-131fefba2e2f","remote_addr":"[fd00:1122:3344:101::1]:58522","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8700,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:01.315890776Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:01.567083493Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:01.567189625Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:34457"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:02.576139408Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"5e74c4de-5d44-4901-981e-764a6e4066be","remote_addr":"[fd00:1122:3344:101::1]:34457","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1008840,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:02.579194119Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:02.579204349Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:02.869999449Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:02.8701258Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:62616"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:02.877692337Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"b74adafc-44be-45ef-894e-d0f997061436","remote_addr":"[fd00:1122:3344:101::1]:62616","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7411,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:02.880848249Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:02.880857119Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:03.058507391Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:03.058635812Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:03.066884706Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"79a97bfd-a60e-498b-80df-156c12cca403","remote_addr":"[fd00:1122:3344:101::1]:64483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8105,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:03.069573303Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:03.069581173Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:03.485884736Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:03.485995057Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:63281"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:03.494824797Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"4f10328c-07fc-48d4-99b8-0c5fd41bf00b","remote_addr":"[fd00:1122:3344:101::1]:63281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8726,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:03.496808517Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:03.496816647Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:03.921627166Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:03.921731277Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:32975"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:03.929474615Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"df156a8b-e0f0-4c78-9668-33988319af97","remote_addr":"[fd00:1122:3344:101::1]:32975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7640,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:03.931295784Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:03.931301054Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:04.452318968Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:04.452420149Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:58389"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:04.460620192Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"efdf8c14-66b1-43e1-8098-800d06f3045a","remote_addr":"[fd00:1122:3344:101::1]:58389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8097,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:04.463681123Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:04.463690153Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:05.168646894Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:05.168767925Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55836"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:05.177570064Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"c5e6a84d-00a9-4d18-9eab-b23f6c2e6ff5","remote_addr":"[fd00:1122:3344:101::1]:55836","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8653,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:05.181197111Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:05.181203231Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:05.952330723Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:05.952432094Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:35168"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:05.961086922Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"831406ae-ae95-424b-8bb5-5c81f9856af5","remote_addr":"[fd00:1122:3344:101::1]:35168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8494,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:05.964212984Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:05.964221973Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:06.486794884Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:06.486895325Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:32810"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:06.495893596Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"316e6bbd-782b-4b21-9551-3d34f4d7c1d8","remote_addr":"[fd00:1122:3344:101::1]:32810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8862,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:06.497891106Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:06.497898126Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:06.996853137Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:06.996979658Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.005538795Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"315bf39c-923f-405c-b6ce-8e21f5d3cd8e","remote_addr":"[fd00:1122:3344:101::1]:49631","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8432,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.008716298Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:07.008751218Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.709149843Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.709244964Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55300"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.72958374Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"5be6f122-d8ab-4d8c-b015-98ef961061da","remote_addr":"[fd00:1122:3344:101::1]:55300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":20197,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1209934681, ip_addr: 72.30.35.89, stratum: 3, ref_time: 1718659025.8811648, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.73150062Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.73150574Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1224"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.759422053Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.759556014Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49683"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.759855337Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.759865997Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.782113233Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.782168483Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.801591611Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.801628881Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.820013457Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.820055808Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.879578332Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.879621242Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.936157405Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.936199086Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.991245774Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:07.991279874Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.043996879Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.044043639Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.013724089Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.01377378Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.014063512Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.014201524Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.096880183Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.096912703Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.096916523Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.096920133Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.096923883Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:08.140427224Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"277.606945ms"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.187911256Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.19226165Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.196261511Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.200408613Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.20512865Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.209927449Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.214590566Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.218949901Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.222924071Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:08.227064813Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:11.403710199Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:11.40386508Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:11.410401667Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:11.410629889Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:11.410658949Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:11.416898073Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:11.46181077Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:11.461933271Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:11.540480499Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:11.540630781Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 68.381138ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:11.555697464Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578\") failed: Property not found. retry in 54.766033ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:11.5621854Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 33.178609ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:11.56906321Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 47.889085ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:11.606696212Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 143.009119ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:11.614575313Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578\") failed: Property not found. retry in 104.582182ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:11.761623999Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 201.220125ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:12.282982301Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 58.480394ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:12.568310263Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451\") failed: Property not found. retry in 35.535938ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:12.615646604Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 96.818174ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:12.670615143Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578\") failed: Property not found. retry in 257.432253ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:12.736235631Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 514.078472ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:12.876144394Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 117.763197ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:13.315463932Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451\") failed: Property not found. retry in 94.824497ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:13.522008192Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 149.842059ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:13.706290467Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578\") failed: Property not found. retry in 514.468536ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:14.115254446Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 485.205277ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:14.267908788Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 210.589604ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:14.309601973Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451\") failed: Property not found. retry in 298.842611ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:14.377295621Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 476.492752ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:14.40277931Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 881.682243ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:14.537418679Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578\") failed: Property not found. retry in 632.427934ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:14.639468657Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451\") failed: Property not found. retry in 392.160507ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:14.686418893Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 1.345985933s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:14.767356938Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 712.293953ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:14.890103137Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451\") failed: Property not found. retry in 1.00645282s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:15.08911273Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578\") failed: Property not found. retry in 1.410585368s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:15.279641518Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 969.21467ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:15.425926666Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 910.826251ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:15.609424682Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451\") failed: Property not found. retry in 1.458444865s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:16.102325755Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 984.276363ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:16.11860218Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 946.182575ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:16.40271317Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 1.444442303s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:16.527114666Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578\") failed: Property not found. retry in 611.54379ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:16.69472526Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 944.008884ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:17.110495428Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578\") failed: Property not found. retry in 863.723878ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:17.319178681Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 1.250668391s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:17.357331389Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451\") failed: Property not found. retry in 1.177482619s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:17.571190344Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 654.878023ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:17.977266964Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 1.117578945s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:18.06152336Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578\") failed: Property not found. retry in 500.9451ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:18.190203839Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 1.054779419s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:18.613498585Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 1.322483727s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:18.639045574Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578\") failed: Property not found. retry in 709.231024ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:18.698052074Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451\") failed: Property not found. retry in 1.344456201s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:18.753059964Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 1.297674757s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:19.184410501Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578\") failed: Property not found. retry in 823.215435ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:19.414886495Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 813.716667ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:19.675506936Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 1.017103943s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:19.96694285Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451\") failed: Property not found. retry in 684.284509ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:20.103283696Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578\") failed: Property not found. retry in 1.172009827s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:20.245627084Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 919.771201ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:20.489282152Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 727.875162ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:20.497517456Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451\") failed: Property not found. retry in 574.115424ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:20.797429366Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 1.059142611s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:20.991755693Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 889.417695ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:21.236209839Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 657.10554ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:21.414681324Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 1.153343022s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:22.0574169Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 1.238772611s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:22.079061471Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 1.070667534s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:22.133023549Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 1.434348295s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:23.211940752Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 563.222032ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:23.222632851Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 643.462463ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:23.324440806Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:23.410189329Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45308"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:23.41033996Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"848fc28e-6c69-4f7d-a7a5-bc46c6eaa4c8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45308","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":82,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:23.671657108Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"291.024574ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 1.393519145s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:23.794925032Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 971.074308ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:24.005794876Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 1.403433869s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:24.652468013Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 1.162483407s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:24.985667152Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 1.193772424s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:25.194193953Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 1.27238226s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:26.061356572Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 1.484802549s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:26.153139865Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 818.182631ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:26.393174947Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 1.453177488s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:27.217220168Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 1.227101746s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:27.338948166Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 1.050781449s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:27.642286231Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 1.232900858s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:28.572934651Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 875.325607ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:28.675083135Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 630.495016ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:28.699915248Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9\") failed: Property not found. retry in 1.458854528s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:29.339061418Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a\") failed: Property not found. retry in 1.436135894s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:29.557481949Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 919.317093ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:29.811609176Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a\") failed: Property not found. retry in 826.694507ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:30.7384653Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:31.58501531Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_cockroachdb_e6c421b3-e2dd-444e-99cd-09996d135578"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:31.5850774Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:31.585084971Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_cockroachdb_ed473871-f330-40e6-bd96-bbfc2c4e0f4a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:31.585111091Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_cockroachdb_86a7ca7d-a285-492d-a235-6ce42909a451"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:31.585115361Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_cockroachdb_faa5ef6e-911b-4e07-900d-997c4f84c67a"} {"msg":"Writing ledger to /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:31.585124481Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:31.585466744Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:31.585742777Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"66623f96-8439-48d3-aec2-faea27364471","remote_addr":"[fd00:1122:3344:101::1]:49683","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":23734347,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:31.61649533Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:44346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:31.64504604Z","hostname":"sock","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"f8c8e281-96bc-4c31-bbd5-aa9807315036","remote_addr":"[fd00:1122:3344:101::1]:44346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_3a5b0f31-6281-4af8-b294-58d642ae9ce9 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":28474,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:17:31.645282813Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:966","retry_after":"80.998929ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f8c8e281-96bc-4c31-bbd5-aa9807315036\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:17:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f8c8e281-96bc-4c31-bbd5-aa9807315036\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:31.758611605Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3467"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:17:47.331188704Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3485"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.249863204Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3508"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.25040314Z","hostname":"sock","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"fb006cf1-4e0e-430b-aca5-49b1c862046f","remote_addr":"[fd00:1122:3344:101::1]:44346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":32610955,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.250947336Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36387"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.251592682Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"2a35a911-91f8-4875-8858-dc08d2e95dcf","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36387","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":266,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:04.257857896Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"1.123417468s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.289943062Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.290121554Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:35909"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.291406177Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.291412377Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.314405791Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.314451511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.337149082Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.337197923Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.359057245Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.359105485Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.38217704Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.382227571Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.405668259Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.405719259Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.426128147Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.426174048Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.446448984Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.446493524Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.467218305Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.467287246Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.555868127Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.555919427Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.660908385Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.660957166Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.753135423Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.753194451Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.846110059Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.846165619Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.941746821Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:04.941804701Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.033760777Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.033819007Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.132474931Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.132530591Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.213314203Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.213364363Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.303283758Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.303344808Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.391032531Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.391077961Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.472791272Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.472841632Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.557368752Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.557438963Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.557745956Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.558070589Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672473333Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672535383Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672546474Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672555754Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672562564Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672582404Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672589164Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672594034Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672602064Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672610404Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672616644Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672624704Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672631404Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672639285Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672645905Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672654005Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672660435Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672668225Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.672676125Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.672796856Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"72d5c4f1-82c1-408b-bb53-2df48194cfe7","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized)]","latency_us":1386355,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:05.673067409Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"111.141873ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"72d5c4f1-82c1-408b-bb53-2df48194cfe7\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"72d5c4f1-82c1-408b-bb53-2df48194cfe7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.784972377Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.785937327Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.785948327Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.808281784Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.808326326Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.829196137Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.829244708Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.849629945Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.849668225Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.868687519Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.868724979Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.887972175Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.888018125Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.907335702Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.907383642Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.92684509Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.92688936Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.945823563Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.945865973Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.967024458Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.967066869Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.987861595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:05.987896331Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.008103096Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.008140247Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.028714166Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.028751092Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.049422806Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.049471757Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.069112247Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.069154027Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.08809139Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.08813206Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.107728189Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.10777763Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.126890274Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.126937555Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.145705746Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.145750276Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.165637578Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.165683099Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.184757383Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.184810943Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.185041376Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.185226158Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283507407Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283552498Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283560968Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283566928Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283574188Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283598038Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283605518Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283611268Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283617038Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283624149Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283628578Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283635769Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283641149Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283648389Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283654379Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283659919Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283665339Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283672419Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.283677669Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.28382552Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8a7999af-a839-419b-a070-51a98f7559fb","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized)]","latency_us":500005,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.28406651Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"159.881182ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8a7999af-a839-419b-a070-51a98f7559fb\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8a7999af-a839-419b-a070-51a98f7559fb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.444809258Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.445137841Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.445145261Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.468825242Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.468873813Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.487410201Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.487457522Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.508374204Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.508417805Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.528001704Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.528039734Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.548146109Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.548189869Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.56792219Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.56798135Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.586737561Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.586783572Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.605678054Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.605733654Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.625821599Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.625873789Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.645482179Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.645531189Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.663889346Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.663932937Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.682144122Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.682186572Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.701765571Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.701814962Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.722842686Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.722882188Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.744390315Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.744436275Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.765805993Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.765859023Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.784165269Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.78420622Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.801768768Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.801817799Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.820425418Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.820469949Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.839526783Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.839581114Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.839806305Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.839939527Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937791282Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937838722Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937843872Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937848082Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937852042Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937863322Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937867452Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937871503Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937875163Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937879092Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937883003Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937886852Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937890703Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937894413Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.937993454Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.938000684Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.938010704Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.938020324Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.938031644Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:06.938189906Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"495ca94b-6abf-4e85-b82c-462e0587118b","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized)]","latency_us":494575,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:06.93863373Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"139.821958ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"495ca94b-6abf-4e85-b82c-462e0587118b\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"495ca94b-6abf-4e85-b82c-462e0587118b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.078682444Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.079047358Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.079055068Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.103126823Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.103175974Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.12349652Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.123537951Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.143343722Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.143393173Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.16276045Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.16279861Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.182224008Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.182266258Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.201321622Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.201363382Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.220132123Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.220172124Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.238704692Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.238752923Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.258813197Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.258856907Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.278904081Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.278946141Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.299085126Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.299124387Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.319427453Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.319465744Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.339617118Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.339668539Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.358737073Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.358773593Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.376912528Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.376954878Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.397199444Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.397239265Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.416616612Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.416658072Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.437435773Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.437482074Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.457478577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.457519238Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.478199038Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.478247448Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.478454401Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.478586472Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.575875061Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.575937632Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.575944682Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.575950422Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.575957782Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.575989873Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.575996043Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576001783Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576008743Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576014413Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576021323Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576028973Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576035863Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576041403Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576048343Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576053963Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576061033Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576066834Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576072274Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.576191035Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"0ae995c6-7690-476b-a0aa-9e8f7b3f4337","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized)]","latency_us":498676,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:07.576542958Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"115.168997ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0ae995c6-7690-476b-a0aa-9e8f7b3f4337\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0ae995c6-7690-476b-a0aa-9e8f7b3f4337\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.692443617Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.692799391Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.692806661Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.715777495Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.715832635Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.735888499Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.735931069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.757497459Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.757539189Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.776740935Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.776782385Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.794668387Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.794706187Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.812282486Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.812320816Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.833858626Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.833899836Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.853282793Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.853322973Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.871607309Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.87164425Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.889817834Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.889857095Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.910289672Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.910342703Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.928872801Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.928916002Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.946951335Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.947001456Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.965991109Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.966029819Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.984950002Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:07.984988902Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.003201907Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.003245078Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.023767936Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.023811207Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.04473541Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.0447751Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.063905005Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.063954225Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.082639655Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.082692696Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.082923678Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.08311157Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181219778Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181278858Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181285959Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181292319Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181298109Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181303529Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181320759Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181328109Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181333199Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181339869Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181345399Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181350729Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181357909Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181363379Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181370079Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.18137525Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181382519Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.18138792Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.18139439Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.181495971Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"fe8d7a1c-f3a1-435c-b5ea-cd457ab5eec2","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized)]","latency_us":490206,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.181722863Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"124.14997ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fe8d7a1c-f3a1-435c-b5ea-cd457ab5eec2\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fe8d7a1c-f3a1-435c-b5ea-cd457ab5eec2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.306625324Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.306938037Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.306945497Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.330259224Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.330305694Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.350997955Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.351036925Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.371415316Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.371455733Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.389479346Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.389517367Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.408959174Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.408995345Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.429839857Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.429873707Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.449477556Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.449513887Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.467799773Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.467838553Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.487588774Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.487627564Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.506480036Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.506513006Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.526625201Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.526663161Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.547444293Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.547479303Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.566399155Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.566446846Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.588353429Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.588389309Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.607921908Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.607957768Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.626999022Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.627033402Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.646712852Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.646751313Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.665267631Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.665304821Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.683937261Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.683975851Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.70246557Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.702517085Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.702712772Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.702847383Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798342755Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798401475Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798408735Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798415515Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798419735Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798425346Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798446126Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798451746Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798458836Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798464006Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798469656Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798476806Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798482436Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798489116Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798494726Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798501826Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798505996Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798509876Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.798513616Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:08.798602007Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"08687fc5-8c5c-49c1-8173-f020148b1285","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized)]","latency_us":493164,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:08.79882741Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"287.677394ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"08687fc5-8c5c-49c1-8173-f020148b1285\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"08687fc5-8c5c-49c1-8173-f020148b1285\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.086319183Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.087594086Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.087608796Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.111489919Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.11154419Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.132321011Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.132368481Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.152127462Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.152189973Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.171411228Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.171461869Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.189873776Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.189917257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.209832139Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.20987079Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.229479159Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.229517789Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.248795086Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.248841556Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.268204693Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.268241063Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.287792952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.287840972Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.307832256Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.30787009Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.328595647Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.328639687Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.348190786Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.348237577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.366047128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.366091408Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.385361615Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.385398925Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.40653345Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.40657666Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.425374251Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.425408472Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.445666598Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.445715248Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.466424839Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.466462238Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.485510773Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.485556803Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.485759816Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.485877387Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.583908074Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.583963744Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.583970135Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.583975804Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.583981235Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.583985525Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584002375Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584009415Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584015125Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584020275Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584025645Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584031225Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584038055Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584043555Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584050055Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584054755Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584060065Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584063945Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584067816Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.584148696Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"952acd63-3aa1-4766-a349-af82c914c24d","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized)]","latency_us":499014,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:09.584373869Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"210.301847ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"952acd63-3aa1-4766-a349-af82c914c24d\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"952acd63-3aa1-4766-a349-af82c914c24d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.796073052Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.796456596Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.796468696Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.817864914Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.817908174Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.839049409Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.839088229Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.858498697Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.858535167Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.878612811Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.878658012Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.896915387Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.896952268Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.917237841Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.917271154Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.936716322Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.936752862Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.955359132Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.955395742Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.975349905Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.975383125Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.993700151Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:09.993732782Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.013585004Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.013633294Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.031630027Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.031664048Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.049380888Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.049428368Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.071158789Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.07120244Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.089595777Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.089631177Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.107680861Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.107718327Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.12628807Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.12632452Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.144485745Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.144532386Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.162443028Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.162480328Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.18037081Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.180422291Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.180621563Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.180746584Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279472848Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279529009Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279537319Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279542649Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279548169Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279554919Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279581919Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279586069Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279589769Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279593669Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279597429Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.27960263Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279608319Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279613959Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.27961889Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.27962409Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.27962793Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.2796317Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.27963687Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.27968537Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e0ec636d-135c-44ac-950f-c16535649bdf","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized)]","latency_us":484755,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:10.279898502Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"405.214743ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e0ec636d-135c-44ac-950f-c16535649bdf\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e0ec636d-135c-44ac-950f-c16535649bdf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.685344786Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.68574932Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.685762021Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.707536652Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.707577342Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.72702463Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.72706315Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.747421128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.747464148Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.766712984Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.766753354Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.787377874Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.787418654Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.80668539Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.806719961Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.825897196Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.825932666Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.844751581Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.844790868Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.865253576Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.865302296Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.884394821Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.884430711Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.903738757Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.903773447Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.922592449Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.922628809Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.940631212Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.940667062Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.960862248Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.960903958Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.97972637Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.97976631Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.998586252Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:10.998632532Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.017287992Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.017326182Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.03579151Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.035845161Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.056986646Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.057021806Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.077106173Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.077148541Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.077354293Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.077505744Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.1783809Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178441391Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178449941Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178457071Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178462291Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178469551Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178486321Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178492181Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178497592Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178504801Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178512342Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178517532Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178524672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178530172Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178537002Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178542152Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178550852Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178576172Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178584952Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.178724134Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"88b5395d-4a90-4a00-b111-61c9b9d56719","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized)]","latency_us":494540,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:11.178963366Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"426.119601ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"88b5395d-4a90-4a00-b111-61c9b9d56719\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"88b5395d-4a90-4a00-b111-61c9b9d56719\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.605338813Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.605686951Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.605695464Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.629367897Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.629416728Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.6492562Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.64930599Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.6689807Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.669028141Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.688371397Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.688433207Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.706660393Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.706699754Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.724428154Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.724470185Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.744754411Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.744801251Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.764002297Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.764049447Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.782492045Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.782532315Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.804858312Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.804900563Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.824554963Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.824601843Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.843559696Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.843595796Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.86463198Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.86467354Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.884814575Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.884857056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.904379594Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.904425134Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.925296087Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.925342657Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.945970627Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.946018387Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.966792549Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.966841469Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.986814392Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:11.986859453Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.008134509Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.00818406Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.008404192Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.008558054Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.107360879Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.107415849Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.107424619Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.107432109Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.107437909Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.107443779Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.107449689Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.107471269Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.10747719Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.10748489Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.10749045Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.10749755Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.10750323Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.10750874Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.10751575Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.10752152Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.10752811Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.10753395Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.10753936Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.107665571Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"cd2f6ddd-8bbc-4fc7-9df6-6a631d47f419","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized)]","latency_us":503542,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:12.107884714Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"770.184408ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cd2f6ddd-8bbc-4fc7-9df6-6a631d47f419\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cd2f6ddd-8bbc-4fc7-9df6-6a631d47f419\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.877182368Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.877525322Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.877532702Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.899970069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.90002589Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.920835181Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.920878562Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.942775055Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.942820845Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.961361574Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.961410994Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.981401128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:12.981441628Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.001056317Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.001105138Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.021644627Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.021689087Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.042022804Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.042064394Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.063091878Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.063137169Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.083789489Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.083831769Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.103661571Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.103712181Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.123746575Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.123788546Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.144142513Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.144187563Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.162927994Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.162968064Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.183015938Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.183054479Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.202984971Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.203029172Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.224076486Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.224114236Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.244604454Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.244658435Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.265723749Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.2657688Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.288476511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.288526191Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.288753034Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.288895605Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388020123Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388073284Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388081144Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388086364Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388091824Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388100194Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388105814Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388122664Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388130034Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388135294Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388141845Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388147014Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388153954Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388159435Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388164835Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388168635Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388172375Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388175895Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388179595Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.388268126Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e3a51ce6-3fc4-463b-beaf-82fff5b970d8","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized)]","latency_us":512320,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:13.388506628Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"397.493833ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e3a51ce6-3fc4-463b-beaf-82fff5b970d8\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e3a51ce6-3fc4-463b-beaf-82fff5b970d8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.785728198Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.786052532Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.786060532Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.809452179Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.80950593Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.831772697Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.831816537Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.85179407Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.851843211Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.871898135Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.871957645Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.890739276Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.890783227Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.910086233Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.910126503Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.9304587Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.930499421Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.951944299Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.951992739Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.971836111Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.971883831Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.990958615Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:13.990997276Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.011854798Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.011895618Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.032480098Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.032520458Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.053865195Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.053907085Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.077438035Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.077485716Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.099631651Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.099676541Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.122460913Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.122506593Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.143219684Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.143260014Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.16449936Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.164542011Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.183727566Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.183765676Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.20471273Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.20477375Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.204973332Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.205099333Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.306853048Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.306910129Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.306918409Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.306925709Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.306932569Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.306940509Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.306947549Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.306973679Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.30697971Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.30698701Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.30699244Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.30699794Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.30700314Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.30700989Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.30701526Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.30701877Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.30702424Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.30703067Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.30703609Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.307163721Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"0ac91d75-21e3-4b21-85be-1672c5b48c3a","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized)]","latency_us":522709,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:14.308225432Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"410.789694ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0ac91d75-21e3-4b21-85be-1672c5b48c3a\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0ac91d75-21e3-4b21-85be-1672c5b48c3a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.719390884Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.719721108Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.719728548Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.742155576Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.742199007Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.760431742Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.760471162Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.781213113Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.781254583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.80056447Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.80061429Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.821364491Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.821404792Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.842114872Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.842156313Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.859981924Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.860022895Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.877589083Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.877626894Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.89695899Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.89701121Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.917385918Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.917431128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.939843136Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.939887546Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.957404885Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.957444275Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.976371967Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.976426018Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.997604623Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:14.997644804Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.018578357Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.018615567Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.039011264Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.039051135Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.060667425Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.060700865Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.078748669Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.078790829Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.098265557Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.098308778Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.118688615Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.118734935Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.118964278Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.119104929Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.216932149Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.216981315Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.216988025Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.216993755Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.216999515Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217006215Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217011675Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217031815Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217037495Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217042875Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217048015Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217053355Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217060346Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217065935Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217073395Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217078736Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217084016Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217089156Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217095986Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:15.217198847Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e175eb5a-b342-4d38-b479-ffab0a9a85df","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized)]","latency_us":499013,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:15.217395989Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.252058763s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e175eb5a-b342-4d38-b479-ffab0a9a85df\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e175eb5a-b342-4d38-b479-ffab0a9a85df\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.467546092Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.467919417Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.467928188Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.490733499Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.49077699Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.512213708Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.512261288Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.532175251Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.532219961Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.552454567Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.552495197Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.572717633Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.572756514Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.591473014Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.591511214Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.611477088Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.611518108Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.631978966Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.632022946Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.651627436Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.651672586Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.670412977Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.670452217Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.690268209Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.690304889Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.709476044Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.709513414Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.728290125Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.728332916Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.746579071Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.746616462Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.767009459Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.76704582Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.785149474Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.785189194Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.804801244Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.804838334Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.823693246Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.823734956Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.844882811Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.844916512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.8654124Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.865457008Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.865684793Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.865849195Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.95986244Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.959920351Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.959927321Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.959933601Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.959939681Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.959944081Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.959949891Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.959955461Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.959978821Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.959984921Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.959990492Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.959995712Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.960001041Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.960007432Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.960012822Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.960018022Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.960023322Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.960030022Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.960035072Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:16.960164103Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"5d55fb3f-4c74-4b2f-8297-526bb3183241","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized)]","latency_us":493780,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:16.960386425Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"745.254196ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5d55fb3f-4c74-4b2f-8297-526bb3183241\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5d55fb3f-4c74-4b2f-8297-526bb3183241\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.703927638Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.704276682Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.704284962Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.728066844Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.728120523Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.748205678Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.748257709Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.769018885Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.76906447Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.789487398Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.789539579Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.810076578Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.810123748Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.830080531Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.830124362Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.848775523Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.848827712Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.869034157Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.869076898Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.89001253Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.890068061Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.910817742Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.910869642Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.931015367Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.931061648Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.950425294Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.950470325Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.97066988Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.970717051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.990005047Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:17.990043588Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.008575456Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.008617106Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.028392078Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.028429908Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.047462141Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.047501192Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.068655577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.068715968Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.090957404Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.090999074Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.110792876Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.110851176Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.11108524Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.11124022Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208689981Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208742672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208749412Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208756472Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208761962Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208768832Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208774722Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208781912Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.20880007Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208805818Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208811068Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208816618Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.20882168Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208828327Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208833624Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208837177Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208843973Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208849613Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.208856433Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.208990735Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b7aaf015-53fe-474b-b7e5-200ececb2fa8","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized)]","latency_us":506281,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:18.209214497Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"731.901936ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b7aaf015-53fe-474b-b7e5-200ececb2fa8\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b7aaf015-53fe-474b-b7e5-200ececb2fa8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.939587785Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.939915858Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.939926218Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.960451767Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.960492498Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.98334081Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:18.983385691Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.00296011Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.00299945Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.023962943Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.024004524Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.045700504Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.045738905Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.071134943Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.071183764Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.092718072Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.092762703Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.112871968Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.112910638Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.134491757Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.134541168Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.155037286Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.155086257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.176954269Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.17699096Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.198365427Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.198408927Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.217512152Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.217547992Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.23701762Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.237053498Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.258734371Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.258769901Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.27832531Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.278363181Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.298740648Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.298785068Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.320926733Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.320962634Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.34125681Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.341295241Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.361219803Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.361268404Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.361472826Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.361603717Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458028518Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458073979Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458080968Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458086648Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458092039Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458099089Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458104599Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458109869Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458133379Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458140299Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458145849Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458151169Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458158139Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458162359Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.45816826Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.45817578Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.45818176Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.45818738Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.45819278Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:19.458345221Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"85e4e572-32b1-4d3f-9646-24806f63ead2","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized)]","latency_us":520009,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:19.458572993Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.960897914s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"85e4e572-32b1-4d3f-9646-24806f63ead2\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"85e4e572-32b1-4d3f-9646-24806f63ead2\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:20.384555971Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"1.921779988s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.415252114Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.415615008Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.415622668Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.43844723Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.43849311Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.461402783Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.461448214Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.480937812Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.480980437Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.5043142Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.50436071Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.525617186Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.525663767Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.544984473Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.545020554Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.564548832Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.564583383Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.584166422Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.584201242Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.605851912Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.605885873Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.625749025Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.625787605Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.647196243Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.647231133Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.668293207Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.668336088Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.687335981Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.687378681Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.707831589Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.70786644Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.7274995Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.72753859Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.747563244Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.747599694Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.766948691Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.766983481Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.785900123Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.785948434Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.80715274Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.8071874Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.827066672Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.827111843Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.827342845Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.827493797Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.92721106Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927269751Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927277651Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927285231Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927290601Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927295931Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927312861Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927318372Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927324201Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927329652Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927335092Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927340472Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927347072Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927353702Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927359082Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927365722Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927372632Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927377802Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927383232Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:21.927511623Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a40ca97d-145c-4aa6-888a-21ddfc192492","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized)]","latency_us":513482,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:21.927713105Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.311124522s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a40ca97d-145c-4aa6-888a-21ddfc192492\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a40ca97d-145c-4aa6-888a-21ddfc192492\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.234130423Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.234441416Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.234452806Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.256798884Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.256839434Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.27615812Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.276204771Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.296161694Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.296204954Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.314038226Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.314095586Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.335494814Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.335534274Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.353689049Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.353743639Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.374230738Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.374281078Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.392509194Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.392556144Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.414166174Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.414212005Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.432887475Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.432930465Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.451074069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.45112558Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.472207014Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.472249835Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.492172057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.492229578Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.510480754Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.510529394Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.529561578Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.529600558Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.549237398Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.549278698Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.573259942Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.573304352Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.594887842Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.594942083Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.615037017Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.615086297Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.634451921Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.634514775Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.634719637Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.634870249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730711153Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730768964Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730777334Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730782934Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730789654Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730795134Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730820405Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730826275Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730831915Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730837215Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730841335Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730848375Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730852655Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730857915Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730863085Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730868435Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730875165Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730880305Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.730885515Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:24.731014357Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"1eed506f-c5b2-49b5-bcfe-06900864780a","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized)]","latency_us":498086,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:24.731182442Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.58242588s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1eed506f-c5b2-49b5-bcfe-06900864780a\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1eed506f-c5b2-49b5-bcfe-06900864780a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.309787053Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.310804133Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.310816154Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.334659066Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.334692926Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.35669203Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.35672397Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.377892556Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.377925566Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.397571256Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.397613446Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.417991173Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.418024604Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.437719204Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.437755265Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.457175562Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.457209372Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.476558339Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.47659467Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.495635523Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.495670624Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.515336944Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.515373984Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.534434198Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.534463678Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.555440231Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.555472322Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.574885779Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.57491626Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.593698501Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.593733051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.613985397Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.614032978Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.63393188Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.63397841Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.65461843Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.654659141Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.674389901Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.674450362Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.694860281Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.69490017Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.716426849Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.716487429Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.716689422Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.716829543Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.813994921Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814067542Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814076322Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814095122Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814102652Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814109733Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814126843Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814133063Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814138713Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814142993Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814148583Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814158733Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814169123Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814176053Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814180253Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814185523Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814189313Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814193293Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814198303Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:26.814242834Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"eaea2efb-f8aa-4a21-921c-ab1a3447be48","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized)]","latency_us":505651,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:26.814416005Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.192212836s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eaea2efb-f8aa-4a21-921c-ab1a3447be48\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eaea2efb-f8aa-4a21-921c-ab1a3447be48\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.001893033Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.002194586Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.002203636Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.029568026Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.029636595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.052026513Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.052075253Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.072502791Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.072546411Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.091508214Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.091562745Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.111030763Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.111071783Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.131782074Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.131823464Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.151868678Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.151923159Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.173270446Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.173316326Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.193590882Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.193638193Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.215784088Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.215840799Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.235017964Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.235067684Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.253007727Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.253052377Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.273093861Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.273150562Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.292910483Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.292956213Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.31380136Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.313845416Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.332789168Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.332846339Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.353403578Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.353449888Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.373137629Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.373188009Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.393270724Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.393323504Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.412831692Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.412903593Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.413106865Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.413245356Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511320914Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511396345Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511403595Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511421095Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511427665Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511435045Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511452695Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511479756Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511485656Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511492686Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511498366Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511505616Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511511296Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511520196Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511525926Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511530266Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511540246Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511546096Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511551567Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:29.511620847Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a5aa102a-1eb1-4c83-8de4-954abfcf2e72","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized)]","latency_us":510985,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:29.511817569Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.16782723s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a5aa102a-1eb1-4c83-8de4-954abfcf2e72\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a5aa102a-1eb1-4c83-8de4-954abfcf2e72\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.674887879Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.675249053Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.675258673Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.698890073Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.698942583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.719907227Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.719968597Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.741102522Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.741152683Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.761041365Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.761098666Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.781021218Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.781086799Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.801697779Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.801745849Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.820447309Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.82049573Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.840279931Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.840339272Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.85992487Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.859977331Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.880718202Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.880775892Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.902611514Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.902675605Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.922910131Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.922963591Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.945005445Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.945060846Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.966395493Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.966455034Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.987275735Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:31.987328726Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.009010536Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.009064657Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.029388104Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.029444577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.049826462Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.052208056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.07229804Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.072351134Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.093649407Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.093708598Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.09392416Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.094073952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198411973Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198478523Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198486554Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198492914Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198499934Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198505334Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198512474Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198529864Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198535504Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198546644Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198552454Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198559344Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198564904Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198571744Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198582675Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198586915Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198597325Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198601635Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198607165Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:32.198683396Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"02985068-b5dd-459e-8e1d-fd6b986cce4d","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized)]","latency_us":525070,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:32.198894328Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"5.652734769s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"02985068-b5dd-459e-8e1d-fd6b986cce4d\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"02985068-b5dd-459e-8e1d-fd6b986cce4d\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:34.20300564Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41428"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:34.203101691Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"508e80f2-76b2-431d-9e4d-589c9e50ce68","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41428","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:37.296976388Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"3.451572965s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.837813258Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.838117901Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.838126161Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.861806892Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.861871233Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.881350571Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.881398891Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.903584012Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.903645057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.923981914Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.924042275Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.943125479Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.943170209Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.962460116Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.962506876Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.981803582Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:37.984111816Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.000839236Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.000890336Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.021603847Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.021654258Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.041699391Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.044079106Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.066087759Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.066143965Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.086064665Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.086125063Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.10743451Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.107503561Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.128271412Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.128323252Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.149039023Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.149088744Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.170485741Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.170541952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.191249332Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.191308393Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.212289846Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.212341587Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.235008908Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.235066818Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.256661108Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.256723809Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.256961471Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.257122313Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359509974Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359579995Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359612595Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359626505Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359633105Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359639125Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359644895Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359650675Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359657836Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359665325Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359671025Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359676196Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359687566Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359697506Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359707786Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359717006Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359736016Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359754877Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.359759597Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:38.359848027Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"07e2f455-b00e-4582-b734-b58e32d61313","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized)]","latency_us":523314,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:38.360055129Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"6.114420119s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"07e2f455-b00e-4582-b734-b58e32d61313\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"07e2f455-b00e-4582-b734-b58e32d61313\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.459418343Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.459698346Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.459706926Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.483423267Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.485796721Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.503901395Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.503956836Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.522222952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.522282612Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.541159874Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.541201475Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.558950285Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.558997676Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.576941198Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.576996869Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.594977272Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.595035692Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.612828423Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.612879824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.631326641Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.631366838Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.649617197Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.649666308Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.667822373Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.667870293Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.685524503Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.685576813Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.703862079Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.703904809Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.721245966Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.721291357Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.739693964Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.739737464Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.758420204Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.758462684Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.777134694Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.777203875Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.795289729Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.79534592Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.813804477Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.813852685Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.832433476Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.832488877Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.832708399Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.832859291Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924756395Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924829556Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924860336Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924869336Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924880127Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924886207Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924893376Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924919457Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924928617Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924934477Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924940137Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924949007Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924963917Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924970067Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924981337Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924987197Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.924992957Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.925010728Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.925018038Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:44.925025878Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4da6c1e3-9c2e-4b8a-bfad-68cd6da28293","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized)]","latency_us":466712,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:44.92524937Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"6.892136684s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4da6c1e3-9c2e-4b8a-bfad-68cd6da28293\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4da6c1e3-9c2e-4b8a-bfad-68cd6da28293\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.800563955Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.800936659Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.800952948Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.826480379Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.826537709Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.849540343Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.849593084Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.871594498Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.871647208Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.89345243Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.893511591Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.916357373Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.916403603Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.938611409Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.93867046Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.960253249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.96031036Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.981064321Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:51.981109501Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.002533099Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.00258845Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.025671335Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.025713085Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.047114103Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.047158963Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.067978285Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.07039253Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.090143281Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.090216821Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.112064644Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.11211372Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.132984606Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.133041127Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.15397673Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.15403036Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.175868442Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.175917513Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.197557723Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.197611624Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.218718118Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.218776559Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.239825863Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.239896884Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.240105616Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.240248997Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.339977652Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340051242Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340085913Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340100953Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340105483Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340109703Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340113923Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340118123Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340123803Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340129193Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340140713Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340146223Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340153103Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340162024Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340166253Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340173434Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340177394Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340193634Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340197914Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:52.340205254Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2f4adb22-9404-4a02-a9bf-1654256e5d70","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized)]","latency_us":540939,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:52.340436036Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"6.272114235s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2f4adb22-9404-4a02-a9bf-1654256e5d70\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2f4adb22-9404-4a02-a9bf-1654256e5d70\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:55.735276434Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"4.397233136s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.597363112Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.597893597Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.597906917Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.620141214Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.622570538Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.639495531Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.639552291Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.657744946Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.657795657Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.673066872Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.673110222Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.69057344Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.690629431Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.707183679Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.707226449Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.726468325Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.726515716Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.744262216Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.744322497Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.762585102Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.762629363Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.780713286Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.780766437Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.79871835Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.7987685Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.81548369Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.815528Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.833257641Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.833314581Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.851001801Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.851047632Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.868202766Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.868258577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.886444162Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.886506042Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.904467595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.904525746Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.923857802Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.923904873Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.942565922Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.942676544Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.964050491Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.964137952Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.964416305Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:58.964584255Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060535882Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060611213Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060620873Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060631063Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060636643Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060642703Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060662644Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060668974Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060676984Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060682394Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060689704Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060695754Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060709784Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060715284Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060722314Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060744575Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060748935Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060754415Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.060759995Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:18:59.060827285Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"20c83209-2d4d-4e6f-a069-db06c76a19f0","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized)]","latency_us":464516,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:18:59.061048907Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"5.700143689s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"20c83209-2d4d-4e6f-a069-db06c76a19f0\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:18:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"20c83209-2d4d-4e6f-a069-db06c76a19f0\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.152887744Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35389"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.152951215Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"071b7ef9-48c5-442c-acbf-e48604f83489","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35389","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.747266879Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.748204609Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.748214099Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.770167872Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.772562866Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.788989763Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.789049064Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.808111638Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.808164658Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.825953539Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.82601041Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.843575149Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.843620709Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.861639631Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.861699193Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.880999039Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.88105642Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.898587238Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.898641649Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.917092766Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.917152777Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.934868987Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.934916858Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.953476996Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.953527277Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.972007255Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.972076905Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.990748135Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:04.990806626Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.008832259Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.00888427Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.027227786Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.027286167Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.045520243Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.045574443Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.064163472Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.064242703Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.084512509Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.0845728Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.102853375Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.102898196Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.12097941Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.121049101Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.121248433Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.121386354Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.2134084Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213499121Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213510371Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213518891Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213528931Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213536971Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213545512Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213553512Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213570812Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213581812Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213590302Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213600272Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213626072Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213644942Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213653342Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213663153Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213671244Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213678594Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.213696453Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3060","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:05.213810504Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"5a2bbfc2-476c-49df-821a-f4e0e38dfbb3","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\", TimeNotSynchronized), (\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\", TimeNotSynchronized), (\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\", TimeNotSynchronized), (\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\", TimeNotSynchronized), (\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\", TimeNotSynchronized), (\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\", TimeNotSynchronized), (\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\", TimeNotSynchronized), (\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\", TimeNotSynchronized), (\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\", TimeNotSynchronized), (\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\", TimeNotSynchronized), (\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\", TimeNotSynchronized), (\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\", TimeNotSynchronized), (\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\", TimeNotSynchronized), (\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\", TimeNotSynchronized), (\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\", TimeNotSynchronized), (\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\", TimeNotSynchronized), (\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\", TimeNotSynchronized), (\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\", TimeNotSynchronized), (\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\", TimeNotSynchronized)]","latency_us":467659,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:05.214043977Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"14.192387381s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5a2bbfc2-476c-49df-821a-f4e0e38dfbb3\", \"content-length\": \"124\", \"date\": \"Mon, 17 Jun 2024 21:19:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5a2bbfc2-476c-49df-821a-f4e0e38dfbb3\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:15.117692841Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"8.26664595s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 91162038-95e7-4445-b7b9-254e840c7a43, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 05af6327-e353-49c1-8e4c-eb7263dada76, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 94929592-0bbe-41e0-92fe-48d07a3d4330, kind: Service { id: 7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4 }, name: Name(\"ntp-7ff3ea72-7629-4a77-8aaf-cc7d4a6640f4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 133, 78])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e6c421b3-e2dd-444e-99cd-09996d135578, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 86a7ca7d-a285-492d-a235-6ce42909a451, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: faa5ef6e-911b-4e07-900d-997c4f84c67a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: cd88d589-b585-424a-9bd6-4d9a048238bc, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 267680bb-9aa5-4914-b636-f4328bb9d704, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7a9c1ed2-4101-4d85-aff6-13c9c39996df, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: f97fb272-7b79-460f-b22d-3fcab84e4b06, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5833dbb2-a361-415c-8052-f92d0b9fcd67, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 1fde5da3-ef6c-40a3-ac00-304728f8b373, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: cef9099d-909c-4c18-b02d-d985d66287e7, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 827137e4-479a-43b5-ac85-00e876901c1e, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External } } } }, OmicronZoneConfig { id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External } } } }, OmicronZoneConfig { id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External } } } }, OmicronZoneConfig { id: 94b7e2ec-a500-4eca-8de7-e83621463f70, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External } } } }, OmicronZoneConfig { id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1084a748-5a03-40ef-9e28-8ef28665b46e, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External } } } }, OmicronZoneConfig { id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.369199261Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.369533674Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.369541904Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 91162038-95e7-4445-b7b9-254e840c7a43, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.39234588Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.392395846Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 05af6327-e353-49c1-8e4c-eb7263dada76, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.411085687Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.411143507Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 278c789c-97ef-4df7-ba73-58ef0dc8f8c2, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.430722926Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.43302021Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e6c421b3-e2dd-444e-99cd-09996d135578, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.449412857Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.449462817Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 86a7ca7d-a285-492d-a235-6ce42909a451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.467670592Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.467736963Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed473871-f330-40e6-bd96-bbfc2c4e0f4a, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.485236671Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.485286571Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3a5b0f31-6281-4af8-b294-58d642ae9ce9, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.503545487Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.50583246Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: faa5ef6e-911b-4e07-900d-997c4f84c67a, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.522187217Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.522253037Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.540747445Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.540789166Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.559479486Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.559523286Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f97fb272-7b79-460f-b22d-3fcab84e4b06, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.577463929Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.577517509Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0b5e8443-02ac-474e-b32f-a16d3ceeb899, dataset_name: DatasetName { pool_name: ZpoolName { id: ffe46556-d7d9-435b-85f0-4376aa51f547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.595625084Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.595671914Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 827137e4-479a-43b5-ac85-00e876901c1e, dataset_name: DatasetName { pool_name: ZpoolName { id: 2abfdf07-7cd1-4355-89f0-e7d8ec686d7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.61397616Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.614028951Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e78cae5e-091e-41cf-ba97-4e7ef753afb1, dataset_name: DatasetName { pool_name: ZpoolName { id: e4f51e96-8b35-4542-822a-22c6778e005f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.631759441Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.631815592Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2161d902-cdb2-4fd0-9eb7-9ca02f89eb46, dataset_name: DatasetName { pool_name: ZpoolName { id: d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.650035847Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.650085718Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 94b7e2ec-a500-4eca-8de7-e83621463f70, dataset_name: DatasetName { pool_name: ZpoolName { id: 38ec74ae-3290-4738-828b-35608cdf0b99 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.668950509Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.66898985Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 85b29975-9f6d-4bdf-92b7-f0543321d24f, dataset_name: DatasetName { pool_name: ZpoolName { id: cea92678-6b55-4860-a3b6-57f49d0a71a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.690813072Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.690865782Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 62ac6fb6-94c3-4e51-87b2-d996e807c734, dataset_name: DatasetName { pool_name: ZpoolName { id: ab647d9b-1f8b-4c75-927b-5b8bc52c8141 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.710998707Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.713304241Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1084a748-5a03-40ef-9e28-8ef28665b46e, dataset_name: DatasetName { pool_name: ZpoolName { id: fa78ab30-cb65-4bf2-8677-79fb8f81cd6b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.730517145Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.730574496Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c6119325-4a3c-4298-8c63-fc8c8b6de38c, dataset_name: DatasetName { pool_name: ZpoolName { id: 708ddef6-4ed2-4ac7-bfb0-97e7a7008313 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.749901602Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.749976163Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.750221926Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.750356571Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.863374636Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.86372176Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.911520946Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.911775769Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.911810069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.911835639Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.91189344Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.935708562Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.937931525Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:19.938025956Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.103755021Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.103947813Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.142345314Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.142639857Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.142705197Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.142733518Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.142769868Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.14397333Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.145354224Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.145521636Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.151403026Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.156865552Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.161827602Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.166733672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.17321938Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.17833384Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.182860246Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.307809407Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.313454674Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.318517276Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.323254914Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.329001732Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.333800371Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.338343257Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.343094316Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.348178158Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.354269549Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.359390132Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.364421703Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.371353823Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.378305564Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.38478673Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.391780371Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.397596895Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.403340708Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.408076107Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.414863696Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.477010728Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.545358123Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.616411546Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.672080872Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.728116311Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.72992153Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.730847539Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.877928025Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 234, 132]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.877986426Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.881822205Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:20.881845705Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.069285941Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 175, 9]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.069358832Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.075591715Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.075627166Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.172851035Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 137, 255]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.172911165Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.173503661Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.173529912Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.173886095Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.173901345Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.17828491Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.17830505Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.292175575Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 149, 232]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.292251739Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.295852936Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.295875166Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.39489557Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 197, 92]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.394980334Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.395666831Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.399063606Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.403288788Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.403308499Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.483909308Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.483983429Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"Configuring new Omicron zone: oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.484465504Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.492701018Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.492752298Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.493020721Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.493037891Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.493153562Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.495428615Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.495594667Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.499481587Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.501980442Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.501993262Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Installing Omicron zone: oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.529054268Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.536313581Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.625976753Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.657250861Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.666705908Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.676420276Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:21.691258537Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:22.03465041Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:22.044357208Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:31.92314209Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:31.962152587Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:32.18757633Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:32.233763979Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:32.646289195Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:32.653357067Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.341508435Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.341679967Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.380904856Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.381106968Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.403463366Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.457387724Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.457607976Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.464918251Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.477177743Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.4775922Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.484218227Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.484771753Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.485124526Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.491591982Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.499177219Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.499785996Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.585672169Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.588462227Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.895820963Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:33.896145696Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:34.182213046Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:34.182358657Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:34.242217156Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:34.242436348Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:34.516201363Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39095"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:34.565692466Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"dbd5f209-3398-4cf7-a5a8-c75b13cba853","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39095","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47867,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 44.375812ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.597057445Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 68.076121ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.607545922Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 68.958055ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.615379842Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 45.050608ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.623510934Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 73.932303ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.632617417Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 66.029995ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.642325416Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 46.091004ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.649706521Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 74.952451ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.657444929Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 71.066699ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.66537886Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 117.798599ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.67423787Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 55.464604ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.682794237Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 41.952166ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.691155942Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 40.703738ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.699718209Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 70.772765ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.707345797Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 57.29755ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.717069225Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 71.918282ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.725043186Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 110.328062ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.732785477Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 70.941205ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.740996119Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 125.599155ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.749217162Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 123.97287ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.757035322Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 85.626009ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.764837231Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 53.988237ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.778202607Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 82.425308ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.78833109Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 92.705408ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.799928598Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 145.516378ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.812202713Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 136.818066ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.823455267Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 86.956121ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.834773233Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 122.346689ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.843000086Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 105.686444ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.851506113Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 89.006409ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.860563845Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 287.588174ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.87285581Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 120.361146ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.881245195Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 161.902525ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.88863209Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 126.112196ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.895384089Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 188.795436ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.903234079Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 152.580329ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.912977718Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 210.58129ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.920672896Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 251.066742ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.930644588Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 173.844451ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.938679849Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 191.625681ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.948294417Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 219.794651ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.961047637Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 125.213907ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.972431303Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 201.758276ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.981921039Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 133.085153ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.990943861Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 131.323376ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:34.999339516Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 383.9124ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.012708212Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 448.882586ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.032061329Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 376.995322ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.059847122Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 425.360228ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.075114407Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 572.111799ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.101708297Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 406.908635ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.111397456Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 249.839016ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.121885543Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 377.320977ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.134631372Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 254.135724ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.144987968Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 223.499837ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.155225622Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 422.336441ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.165941061Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 375.914382ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.175272046Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 311.684014ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.192160368Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 272.167853ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.201581513Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 421.887511ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.211571035Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 862.744496ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.384899448Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 601.603839ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.395744388Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 1.118827097s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.408898272Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 432.578621ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.420167136Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 913.828743ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.467036313Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:35.468110104Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:35.468267966Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 551.0885ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.510824469Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 979.675629ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.52079796Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 962.901157ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.531681101Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 633.695311ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.542711943Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 618.29212ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.551828446Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 842.898097ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.561851828Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 940.529228ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.57193069Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 577.13197ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.602560872Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 528.772581ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.643411557Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 72.191002ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.652701672Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 1.195241444s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.680280082Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 68.819751ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.735874747Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 205.284761ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.816908571Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 1.423024103s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:35.862544815Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 763.208218ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.005636701Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 562.827504ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.033456684Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 740.67711ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.071177246Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 730.160479ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.178801262Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 1.360880391s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.187846856Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 1.114197929s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.196510112Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 740.327885ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.205905128Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 541.136273ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.256016468Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 555.974007ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.713401139Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 698.523487ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.766333747Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 605.870225ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.839456511Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 666.491535ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.890349479Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 902.13993ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.916238672Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 1.290319941s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.944792622Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 1.012295276s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.972755987Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 1.131428386s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:36.996401107Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 1.160892769s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.017020176Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 1.426928478s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.035741216Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 1.240146478s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.053240874Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 1.409341448s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.071640921Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 623.102642ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.089514482Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 562.503489ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.301719204Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 807.643674ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.319918469Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 1.229471192s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.33382953Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 1.034047694s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.463899259Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 977.768404ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.479434207Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 742.083402ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.55865805Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 653.424564ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.569661442Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 765.688878ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.727047228Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 1.35881307s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.832944102Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 989.742669ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.876443833Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 1.110516563s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:37.995794744Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 561.93238ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.138577442Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 539.803751ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.147018478Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 782.636577ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.187463898Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 1.187517878s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.23107786Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 1.307873096s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.244100302Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 1.306659801s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.302327233Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 1.15526359s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.310319224Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 873.140014ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.473327287Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 1.028606355s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.486530152Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 957.477599ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.499438943Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 1.426233939s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.510798268Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 1.246315986s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.523338115Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 662.454319ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.577435514Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 1.446232329s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.699966026Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 886.70716ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.713271891Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 1.402176283s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.879872921Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:38.991270221Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"26.752023885s"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 717.169169ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.000569575Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 504.012323ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.116192968Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 859.96659ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.200583974Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 640.495456ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.249235537Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 1.225811523s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.355934319Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 926.365429ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.428612047Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 625.351327ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.466656773Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 928.901088ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.474479182Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 1.131464043s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.524217686Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 954.624651ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.560417094Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 1.126109045s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.607828445Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 873.405523ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.61818477Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 828.272942ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.628203631Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 1.354166887s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.726042283Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 926.096421ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.779349674Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 1.352202428s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.899173589Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 720.247754ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:39.945481049Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 1.183821891s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.069286255Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 990.255534ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.101025087Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 1.488189791s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.155799892Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 1.149988007s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.291732201Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 1.375484287s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.363912003Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 1.190285084s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.414103582Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 1.235169783s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.466690086Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 759.820001ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.501003374Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 827.015614ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.562307095Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 939.38251ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.661987377Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 889.021045ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.670518013Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 1.01333921s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.679256832Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 1.230660267s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.713858512Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 660.660295ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:40.742810596Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 895.676248ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.089977967Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 1.125043569s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.101757297Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 711.369029ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.265817881Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 1.025354369s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.274663021Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 1.417325713s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.28247808Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:41.303600684Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:41.321208123Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:41.388663897Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:41.388875959Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 886.130907ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.404258105Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:41.404460447Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 1.231965372s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.413648381Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:41.413769002Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 1.345541378s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.455423534Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 61.600921ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.563284578Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 1.145253852s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.575721285Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 65.756052ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.586801297Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 801.357041ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.648828476Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 92.484524ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.678604557Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 1.297331851s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.69266767Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 1.100059478s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.704173207Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 1.283715486s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.715868255Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 57.193597ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.725482683Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 69.468071ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.738365923Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 1.454341955s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.748194553Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 1.335385205s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.757200774Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 190.218074ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.780108657Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 178.10447ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.795406662Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 73.905262ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.817712358Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 203.447041ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.903367037Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 856.943794ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.954400355Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 449.230233ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.978429489Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 962.90399ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.987512771Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 373.438067ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:41.995644469Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 1.334893529s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.003741565Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 267.336743ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.118378688Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 780.90685ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.238083952Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 740.490923ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.302812559Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 989.408689ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.312425516Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 899.386194ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.380294885Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 794.770685ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.399618031Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 613.838698ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.438564102Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 841.587391ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.460321277Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 1.465448193s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.657121803Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 862.933777ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.712465584Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 755.943605ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.731898581Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 1.188044375s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.81298045Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 1.382757514s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.824717312Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 860.373389ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.836099368Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 801.615444ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:42.974757434Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 1.081860142s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.009948131Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 805.130231ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.023375527Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 585.2995ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.034262718Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 980.40081ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.054391802Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 820.375883ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.064195821Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 998.477352ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.105689572Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 1.220799558s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.207853959Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 876.317396ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.219939181Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 929.759091ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.293435096Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 768.761134ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.313596151Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 754.872775ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.323493272Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 880.350249ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.351420585Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 829.195757ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.500225514Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 1.278521815s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.591072366Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 1.472424979s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.633436355Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 587.68954ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.714093203Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 525.532376ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.787965572Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 1.159765185s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.842218373Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 1.481487173s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:43.898808336Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 522.798131ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.019986826Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 1.247678257s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.060424626Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 1.021020431s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.098935557Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 1.32096342s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.119317013Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 1.139493277s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.136487457Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 966.908373ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.154995515Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 535.913572ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.171380001Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 687.815074ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.186638076Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 1.121128309s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.224252237Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 835.432259ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.238840405Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 1.447578193s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.251534594Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 546.3491ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.315585604Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 772.567975ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.331888059Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 1.418227756s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.347425527Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 1.249398364s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.453789726Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 1.482875753s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.569471039Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 721.665678ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.73034941Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 1.385532013s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.875620994Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 944.02157ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.891777074Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 516.848413ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:44.908909152Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 1.269508363s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.017652884Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 629.405823ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.094730336Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 814.230699ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.122463238Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 1.292950979s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.137382839Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 679.916491ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.154994288Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 557.55218ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.172556606Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 1.39508489s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.289117938Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 1.150076377s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.32182717Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 740.885626ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.355492521Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 688.23107ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.390353915Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 1.199606704s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.43717936Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 1.316470698s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.450809718Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 764.582198ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.461926631Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 705.914412ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.710515602Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 1.161440162s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.718902947Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 1.195514415s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.734692257Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 539.128687ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.743198068Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 624.065272ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.776738134Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 584.685618ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.843950305Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 842.584994ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.85324628Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 1.329887373s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:45.948259403Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 1.072863919s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.063358351Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 617.778664ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.087507716Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 685.801601ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.105319407Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 1.398717623s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.239091953Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 1.478860675s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.272741785Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 1.360291116s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.291483025Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 1.087074497s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.302169673Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 1.059056933s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.412928507Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 827.875936ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.438747739Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 1.36520243s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.451899492Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 812.182315ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.464839643Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 1.407713116s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.482932337Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 549.169838ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.648188403Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 1.028921467s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.695103638Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 696.425012ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.705363292Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 755.077099ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.714434394Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 1.453876123s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.774874817Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 574.608085ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.798544087Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 1.438546451s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.889010005Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 1.324606217s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:46.939706889Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 1.20571866s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.146994512Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 1.217249211s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.207653317Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 671.834936ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.276507564Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 709.855183ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.286109403Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 1.338189187s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.295422767Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 523.087787ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.383734913Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 699.706442ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.398395252Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 1.457069598s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.410295832Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 1.000537783s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.480587216Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 719.122613ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.489344474Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 711.875659ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.651502903Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 974.490958ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.664664683Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 805.687587ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.733223698Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 950.117161ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.761554715Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 767.483579ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.826541594Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 1.386961139s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.901660316Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 1.083612206s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.91880301Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3\") failed: Property not found. retry in 1.275609926s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:47.959728795Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 532.01199ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.01146641Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 584.58623ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.10905883Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 683.099219ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.22341376Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 525.468956ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.242396272Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 664.477916ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.275963663Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 794.730512ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.341223745Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 853.899542ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.366063773Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 1.15798119s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.378959848Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 896.029727ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.438657803Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 1.267248426s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.500639642Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 1.11440264s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.551300026Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 971.011686ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.565939694Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 543.920348ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.608635647Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 958.618757ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.648057137Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 1.200073412s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.66218275Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 1.471364257s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.706875333Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 719.329715ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.728782105Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 618.722281ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.781615641Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 1.407724144s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.879767037Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 1.340946624s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.917485429Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 1.332878977s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:48.95303981Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 1.27288395s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.022344433Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 789.735225ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.152107529Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 546.811253ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.16698855Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 1.167465236s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.234235232Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 736.798595ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.304830068Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 995.581524ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.348493391Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 506.886813ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.416883955Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 1.464915211s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.463178364Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 1.215693491s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.553490501Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 1.063257137s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.565660224Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 505.558069ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.616664441Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 1.096647424s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.674891681Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 1.269022678s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.723982459Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 1.152377546s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.777334131Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 595.419459ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.871352994Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 1.261686996s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.933646326Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 1.174744433s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:49.950692119Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 947.694393ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.051695774Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 860.318869ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.135054539Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 1.156330746s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.188659423Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 1.488826509s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.271957758Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 962.538224ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.297027522Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 1.098928278s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.305617769Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 701.311335ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.31361989Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 519.507431ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.353659216Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 842.817878ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.413719606Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 776.399708ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.47981649Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 962.237547ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.641519136Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 1.291091864s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.780199612Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 591.805236ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.789382636Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 1.363601633s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.886562801Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 1.30634663s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.93973281Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 1.390679173s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:50.949607801Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 1.407461377s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.004229185Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 884.326247ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.014687201Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 1.49198953s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.024453285Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 572.885078ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.034258229Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 1.354716082s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.136452316Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 1.445953855s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.207032452Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 560.042063ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.267532815Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 1.391879286s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.277067012Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 978.393046ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.286333126Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 765.490822ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.355311056Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 918.075202ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.390908517Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 1.01742481s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.413631857Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 588.385825ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.614377224Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 790.26992ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.622744058Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 946.917776ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.772021322Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 1.331625309s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.837141043Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 1.387891256s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:51.90979622Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 1.494981844s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.080728125Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 898.264204ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.130943713Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 1.233436233s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.213966115Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 655.68158ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.25588723Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c\") failed: Property not found. retry in 1.258124594s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.264337456Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 918.990209ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.275973894Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 1.115170535s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.31893935Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 1.297727989s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.349400479Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 1.206448037s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.422619501Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 792.538698ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.430297469Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 722.229004ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.4402265Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 589.164556ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.500854925Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 769.092708ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.525923619Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 1.097760536s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.662737607Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df\") failed: Property not found. retry in 642.918797ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.678461306Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 939.212089ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.728705885Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 1.177407523s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:52.921306949Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373\") failed: Property not found. retry in 1.387853977s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.039486918Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 708.874075ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.098935221Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7\") failed: Property not found. retry in 1.292350615s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.172956381Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 711.110989ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.182855232Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc\") failed: Property not found. retry in 1.116845642s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.206671483Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 816.816143ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.232758888Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a\") failed: Property not found. retry in 1.307322973s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.305605517Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 554.494048ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.31381004Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70\") failed: Property not found. retry in 829.707133ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.443789777Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67\") failed: Property not found. retry in 1.033284608s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.455878101Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e\") failed: Property not found. retry in 1.346482545s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.585484476Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f\") failed: Property not found. retry in 850.003723ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.638342082Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06\") failed: Property not found. retry in 828.797636ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.657020161Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e\") failed: Property not found. retry in 744.964523ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.677811912Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb\") failed: Property not found. retry in 1.397989432s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.768862655Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704\") failed: Property not found. retry in 1.128860236s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.817276366Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1\") failed: Property not found. retry in 699.114825ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.882596459Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899\") failed: Property not found. retry in 1.414841361s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:53.905881175Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46\") failed: Property not found. retry in 1.106929829s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:54.059667265Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734\") failed: Property not found. retry in 1.01703924s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:19:54.107968855Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.348524768Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_clickhouse_f97fb272-7b79-460f-b22d-3fcab84e4b06"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.348606799Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_external_dns_18077c32-d18c-41ec-bf4f-0e30a06c25cb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.348615389Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_pantry_cef9099d-909c-4c18-b02d-d985d66287e7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.348633099Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_0b5e8443-02ac-474e-b32f-a16d3ceeb899"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.348638929Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_827137e4-479a-43b5-ac85-00e876901c1e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.348650679Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_e78cae5e-091e-41cf-ba97-4e7ef753afb1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.348657459Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_external_dns_cadebd1f-a7c1-4f50-9b00-0055fd0e092a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.348663399Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_2161d902-cdb2-4fd0-9eb7-9ca02f89eb46"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.348678159Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_94b7e2ec-a500-4eca-8de7-e83621463f70"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.348684069Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_pantry_1fde5da3-ef6c-40a3-ac00-304728f8b373"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.34869518Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_62ac6fb6-94c3-4e51-87b2-d996e807c734"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.3487031Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_nexus_c34d2d1f-8939-4dc7-829a-e8d77861a9e3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.34871024Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_85b29975-9f6d-4bdf-92b7-f0543321d24f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.34871583Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_1084a748-5a03-40ef-9e28-8ef28665b46e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.34873806Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_c6119325-4a3c-4298-8c63-fc8c8b6de38c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.34874715Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_crucible_pantry_5833dbb2-a361-415c-8052-f92d0b9fcd67"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.34875125Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_nexus_cd88d589-b585-424a-9bd6-4d9a048238bc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.34875519Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_nexus_267680bb-9aa5-4914-b636-f4328bb9d704"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.34875915Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3056","zone":"oxz_oximeter_7a9c1ed2-4101-4d85-aff6-13c9c39996df"} {"msg":"Writing ledger to /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.349463467Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.36153445Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.385124189Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"be578ec4-a10f-4a45-863a-3feea3995838","remote_addr":"[fd00:1122:3344:101::1]:35909","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":36063007,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.399190422Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1244"} {"msg":"Writing ledger to /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.399306643Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.399552165Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.406637847Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:681"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.420929502Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:55.460979119Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:825"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:58.177982816Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:58.29076354Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:58.46719695Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:58.610596944Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:58.784986392Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:58.916726399Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:59.207245155Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:59.428693002Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:19:59.940355701Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:00.547922024Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:01.078381124Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:01.4847609Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:02.520637672Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:03.343361947Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:04.001813015Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:04.595960142Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:04.596591878Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"bb375e2e-1726-444e-90a8-3305a080c0f9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43346","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":57,"response_code":"200"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:04.978705603Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:20:05.780836149Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"80.714498043s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/8e338aae-2fb5-450b-bed3-73c865b83f20/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:06.436401389Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cc19651c-cdcf-44ba-91e7-17e3ddf53210\", \"content-length\": \"173\", \"date\": \"Mon, 17 Jun 2024 21:20:08 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"09996097-89b8-4c39-a1ef-fd8405b108d6\\\"\", request_id: \"cc19651c-cdcf-44ba-91e7-17e3ddf53210\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:08.817009814Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:09.210811859Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:09.386733373Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55059"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:09.387212098Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:09.476992699Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"14d91ae3-2665-4d32-889d-687325bd99a2","remote_addr":"[fd00:1122:3344:101::c]:55059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":90179,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:09.478471854Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:09.478531994Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:09.480733807Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:09.48102961Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"333a4862-d6dc-44b9-bcf4-330800a13254","remote_addr":"[fd00:1122:3344:101::c]:55059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3602,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.510238981Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:45653"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.51213929Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.512299292Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.512334883Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.512355843Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.512362353Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.512378013Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.512397653Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.512428253Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d05c0894-0b7a-4eef-a38a-f2be5bf2c114","remote_addr":"[fd00:1122:3344:101::b]:45653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1982,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.514728877Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:897"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.857379882Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58011"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.857661825Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.857709425Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.857747115Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.857760406Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.857769816Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.857777686Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.857783156Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.857810046Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fcf7ec84-ab8c-485c-8667-d89eb0f87d00","remote_addr":"[fd00:1122:3344:101::c]:58011","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":311,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.883327035Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47419"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.883911651Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.967432508Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"58167ff2-38c9-4fef-895d-d2d4856812ed","remote_addr":"[fd00:1122:3344:101::b]:47419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":84014,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.967523069Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:35404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.967852682Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.967901173Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Reading ledger from /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.967927823Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.967950963Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.968157706Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.968174906Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.968191046Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.968228046Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.968249396Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.968280067Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b5c4a36d-af7d-4e1b-8ae5-39e0b9af2039","remote_addr":"[fd00:1122:3344:101::a]:35404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":675,"response_code":"204"} {"msg":"Reading ledger from /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.968309557Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:11.968566659Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"79b5ac80-ef24-4a4a-b935-6d1b8d65a2c5","remote_addr":"[fd00:1122:3344:101::b]:47419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":799,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:12.209424573Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:46571"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:12.209680765Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:12.209742056Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:12.209794126Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:12.209813037Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:12.209846697Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:12.209854967Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:12.209871737Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:12.209897398Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d8b0b982-9fb7-4015-aa5d-b3e49b3f02ac","remote_addr":"[fd00:1122:3344:101::b]:46571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":392,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:34.630705397Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63707"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:34.630870049Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"a94f7a9f-31a5-43dd-96b2-aed23a1bb22b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63707","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:35.763024801Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:38417"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:35.764107112Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"950de3ee-7af3-44a1-92db-ac9103ff84ac","remote_addr":"[fd00:1122:3344:101::c]:38417","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1063,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:35.938690203Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:63826"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:36.019984854Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"fc3a4335-fe97-4b3e-9642-58e9e8d4f24b","remote_addr":"[fd00:1122:3344:101::c]:63826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":81263,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:36.021346272Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1a968f31-ec93-4fbe-8451-d9505a19c936","remote_addr":"[fd00:1122:3344:101::c]:63826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1102,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:36.043242363Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:36.043278714Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:36.04486276Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:36.04583688Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:36.045959381Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"7f271fbd-bfc3-4416-bb7e-2693aa7f9c8b","remote_addr":"[fd00:1122:3344:101::c]:63826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3558,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:37.86189241Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44421"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:37.862070912Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"66f2cbb7-ddae-4c9b-a190-589df03a0761","remote_addr":"[fd00:1122:3344:101::a]:44421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":165,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:37.9949823Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:42605"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:38.048418342Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"74ebf7a9-9e1b-4a45-af9f-ea13e21f3ed1","remote_addr":"[fd00:1122:3344:101::a]:42605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":53376,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:38.048772786Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"92aaca3f-5b08-4c71-be05-4573781be1cc","remote_addr":"[fd00:1122:3344:101::a]:42605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:38.146141993Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:35296"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:38.146343735Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"327ef22d-4f79-4de2-9b8e-a18f2fe0cb65","remote_addr":"[fd00:1122:3344:101::b]:35296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":161,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:38.236900604Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:42956"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:38.284458326Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"25370edd-bf3c-49c5-b378-b0380eab15e8","remote_addr":"[fd00:1122:3344:101::b]:42956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47506,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:20:38.285529077Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b064f208-e797-4d61-8999-d91a7467116b","remote_addr":"[fd00:1122:3344:101::b]:42956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":865,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:04.663662918Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36930"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:04.664220834Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"939cfe70-c88d-45e7-ae31-b8bbbdb11001","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36930","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":59,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:05.806045155Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47154"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:05.806939014Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"569a6141-9b6e-4d2d-b4ce-76125f72516d","remote_addr":"[fd00:1122:3344:101::c]:47154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":872,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:06.07659898Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54975"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:06.1229886Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"498095a4-9213-45f7-b93e-e83b0df7cac4","remote_addr":"[fd00:1122:3344:101::c]:54975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46302,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:06.123285293Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"bf040bbd-857a-4952-805f-340222ef3b80","remote_addr":"[fd00:1122:3344:101::c]:54975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:07.925065728Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:07.92527072Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"17b3d12b-94f5-406a-a2a1-4a497e1bbad2","remote_addr":"[fd00:1122:3344:101::a]:43855","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:08.25549742Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:52540"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:08.304928091Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"9e37695b-b61a-49bb-abfe-bb50c0a8de88","remote_addr":"[fd00:1122:3344:101::a]:52540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49376,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:08.305231605Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"089239dd-05c3-4634-aec3-903b24a9967a","remote_addr":"[fd00:1122:3344:101::a]:52540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:08.368926531Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38867"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:08.369161743Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"4e323c25-cb1e-4ae6-ae9d-c6b36a916bad","remote_addr":"[fd00:1122:3344:101::b]:38867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:08.553436862Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50455"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:08.602084836Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"9aa85b5f-c2ee-4387-97a2-05c80a1483a7","remote_addr":"[fd00:1122:3344:101::b]:50455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48591,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:08.602357138Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"39c355a7-108e-4677-8cbc-3f4b42dfc436","remote_addr":"[fd00:1122:3344:101::b]:50455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:34.69671518Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64889"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:34.696826352Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"448b901e-bcee-45c4-8601-3be13907f144","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64889","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":92,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:35.72792818Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:39041"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:35.72894604Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"df440212-6335-40a7-a6bb-8eb3fc19dd8e","remote_addr":"[fd00:1122:3344:101::c]:39041","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":999,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:35.837059277Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:39742"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:35.886379167Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"342230eb-047b-4551-a98f-f21a42002d18","remote_addr":"[fd00:1122:3344:101::c]:39742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49243,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:35.8866509Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8a1c8501-cb9b-460e-bfc0-d88c681e1c12","remote_addr":"[fd00:1122:3344:101::c]:39742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:37.882292642Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34152"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:37.882478594Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f4050a45-34f0-4d03-b46f-39308ee59e7c","remote_addr":"[fd00:1122:3344:101::a]:34152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":172,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:38.029880599Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43554"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:38.080912766Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"cc0cb0a6-97f4-4a40-ba92-5ecbe140f5b6","remote_addr":"[fd00:1122:3344:101::a]:43554","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50967,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:38.081170679Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"c7d4755f-cf4e-4aaf-8380-fa363228dd77","remote_addr":"[fd00:1122:3344:101::a]:43554","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:38.100862889Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40870"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:38.101100441Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"2367a230-9a83-4b9b-a290-d9464c21f907","remote_addr":"[fd00:1122:3344:101::b]:40870","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":180,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:38.28635094Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:54471"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:38.333264076Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3fd259c8-2148-4a8b-940e-b6dcf09ac7b9","remote_addr":"[fd00:1122:3344:101::b]:54471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46838,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:38.333546159Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4ef71157-1cf4-4347-9b2c-3778822b87ff","remote_addr":"[fd00:1122:3344:101::b]:54471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:40.713285556Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:56.625172839Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:40617","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::d]:47427"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:21:56.630449783Z","hostname":"sock","pid":1555,"uri":"/09996097-89b8-4c39-a1ef-fd8405b108d6","method":"GET","req_id":"920def1a-6b0e-4ac5-80a5-dc044d5c2627","remote_addr":"[fd00:1122:3344:101::d]:47427","local_addr":"[fd00:1122:3344:101::1]:40617","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":5091,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:04.728055245Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44513"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:04.728899264Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f0373cd1-63a1-4ff3-83b7-1c5cf161dd1c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44513","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":825,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:05.749569277Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:35580"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:05.749769289Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"d63ac5ef-e0bb-4cd7-b230-66ea516d7c39","remote_addr":"[fd00:1122:3344:101::c]:35580","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":181,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:06.119239046Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:33534"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:06.16984062Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"17f40b69-cab5-4783-b920-14ad0cef4489","remote_addr":"[fd00:1122:3344:101::c]:33534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50528,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:06.170086872Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d27927c8-9668-496e-9b1d-1295300ab40c","remote_addr":"[fd00:1122:3344:101::c]:33534","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:08.103516973Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64030"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:08.104273781Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"d3601d08-40a9-40fb-a8b2-a8db7e9d697a","remote_addr":"[fd00:1122:3344:101::a]:64030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":737,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:08.20677892Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43581"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:08.253754927Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"900996fb-bb99-4abe-b379-b78c09192cb4","remote_addr":"[fd00:1122:3344:101::a]:43581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46913,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:08.25405036Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"717adfa6-ec08-41c2-8bc8-4db64e0ab56d","remote_addr":"[fd00:1122:3344:101::a]:43581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:08.481007102Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:36571"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:08.481255984Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f2f8a720-8932-4d05-91c0-6f7f7550dd6a","remote_addr":"[fd00:1122:3344:101::b]:36571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":206,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:08.633147005Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60673"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:08.681771178Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"7696852a-b8f4-4fb6-8e26-9bbce7ba84a2","remote_addr":"[fd00:1122:3344:101::b]:60673","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48549,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:08.68200035Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a6f297c2-15bd-4b5c-aa2d-f7dfd0dac85b","remote_addr":"[fd00:1122:3344:101::b]:60673","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":83,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:26.625938404Z","hostname":"sock","pid":1555,"uri":"/09996097-89b8-4c39-a1ef-fd8405b108d6","method":"GET","req_id":"b3bf9ec7-616e-46e3-bad0-1b1256352db5","remote_addr":"[fd00:1122:3344:101::d]:47427","local_addr":"[fd00:1122:3344:101::1]:40617","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":757,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:34.760242239Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38723"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:34.760878225Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"7a651636-737f-4c34-8c2d-8dbab34d313c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38723","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":623,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:35.783526878Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:62653"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:35.784315376Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"2c84460c-0a2b-40bc-9330-d96fda37bfac","remote_addr":"[fd00:1122:3344:101::c]:62653","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":776,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:35.860164486Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:45512"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:35.910113322Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b636a471-5475-4025-8fab-82cee52ddca0","remote_addr":"[fd00:1122:3344:101::c]:45512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49888,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:35.910405665Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"29e60400-434c-4a68-bc7e-88fb22377929","remote_addr":"[fd00:1122:3344:101::c]:45512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:37.855655876Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:65442"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:37.855870428Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"23854be9-4e1e-4e55-ab37-671c56a6702b","remote_addr":"[fd00:1122:3344:101::a]:65442","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":197,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:37.98616725Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:38042"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:38.041084156Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c447a55e-07dc-47ac-a576-9d18f3229bae","remote_addr":"[fd00:1122:3344:101::a]:38042","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":54849,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:38.041365019Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"23802f2a-8bb1-4b70-8fa9-5208f8ad50da","remote_addr":"[fd00:1122:3344:101::a]:38042","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:38.099439468Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:58961"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:38.099656561Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"a9cf0b34-c32d-4d83-b03c-59c9126c912b","remote_addr":"[fd00:1122:3344:101::b]:58961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":172,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:38.310481659Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:64461"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:38.365015492Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"531f54cd-563c-477b-9df6-f496bb83c605","remote_addr":"[fd00:1122:3344:101::b]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":54434,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:38.366233815Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"da45d3fe-62f2-4b2c-8dba-368fe0d3d52b","remote_addr":"[fd00:1122:3344:101::b]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":976,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:22:56.626077343Z","hostname":"sock","pid":1555,"uri":"/09996097-89b8-4c39-a1ef-fd8405b108d6","method":"GET","req_id":"ef159b85-dc53-462d-a237-2a4afbc25911","remote_addr":"[fd00:1122:3344:101::d]:47427","local_addr":"[fd00:1122:3344:101::1]:40617","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":510,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:04.791914648Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:04.792511924Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"94bc3f82-1cf7-4213-ad49-37b0a3090a5d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64014","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":563,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:05.733938463Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:35074"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:05.734128044Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"860fdcec-5ee9-446e-a3e6-f02a9bdb1248","remote_addr":"[fd00:1122:3344:101::c]:35074","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":177,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:05.96926149Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:38934"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:06.017770422Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b7f31fda-fbab-4868-aac6-757e5fc9534b","remote_addr":"[fd00:1122:3344:101::c]:38934","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48452,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:06.018048924Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f1ffeb36-29c2-4852-b624-02e22794cc01","remote_addr":"[fd00:1122:3344:101::c]:38934","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:07.919085029Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60273"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:07.919258159Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"4a6680d2-2e22-4413-ae48-49c84c302390","remote_addr":"[fd00:1122:3344:101::a]:60273","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":176,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:08.154571965Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:59257"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:08.20731225Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"dfaeac8f-88bb-4714-906b-34c2e5666e5a","remote_addr":"[fd00:1122:3344:101::a]:59257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":52656,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:08.207603743Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"bee1e6d6-a15d-4474-bf50-38146bc77693","remote_addr":"[fd00:1122:3344:101::a]:59257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:08.268549511Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:43927"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:08.268859305Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"344fd7ea-fd38-4dfb-a376-263483626755","remote_addr":"[fd00:1122:3344:101::b]:43927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":254,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:08.519682529Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:62702"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:08.580215593Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"0ff1f1ba-7f36-4169-98b2-5d2917e2895d","remote_addr":"[fd00:1122:3344:101::b]:62702","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":60413,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:08.580497235Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7f670cf2-7ea2-4b5b-ae6d-9b6dc9579910","remote_addr":"[fd00:1122:3344:101::b]:62702","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":116,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:26.626666036Z","hostname":"sock","pid":1555,"uri":"/09996097-89b8-4c39-a1ef-fd8405b108d6","method":"GET","req_id":"4c24724c-7f03-4acf-84cf-875451c3dfab","remote_addr":"[fd00:1122:3344:101::d]:47427","local_addr":"[fd00:1122:3344:101::1]:40617","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":501,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:34.823521286Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42247"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:34.823608287Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"51915648-0d99-43ab-8fbd-a9334b7265b5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42247","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":78,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:35.672077413Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:35.672299075Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f8e7e7c3-b75d-4ac2-8817-1962c062d48b","remote_addr":"[fd00:1122:3344:101::c]:60862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":196,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:35.811423646Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:53722"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:35.860601905Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"431435e1-eac7-4cb0-8fa8-840b0a4a7393","remote_addr":"[fd00:1122:3344:101::c]:53722","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49114,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:35.860857408Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b4c944dc-2f5e-4398-bd19-948460de66b1","remote_addr":"[fd00:1122:3344:101::c]:53722","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:37.864561511Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50647"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:37.864751763Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"28abf782-e6e3-4f43-b751-a8015b222169","remote_addr":"[fd00:1122:3344:101::a]:50647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:37.986256405Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48174"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:38.041846999Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8ba7aaad-fdac-4c63-a71b-45650dc17772","remote_addr":"[fd00:1122:3344:101::a]:48174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":55519,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:38.042186033Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"25c7e8f0-787c-4b06-b1fb-eb838f8967b8","remote_addr":"[fd00:1122:3344:101::a]:48174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:38.158720905Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:57914"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:38.158963427Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"d04c345b-2c4f-4052-bd39-b52e725e94e2","remote_addr":"[fd00:1122:3344:101::b]:57914","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":191,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:38.25878167Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:61182"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:38.310758897Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d99c7de2-198f-40e5-a608-58b5431bad4b","remote_addr":"[fd00:1122:3344:101::b]:61182","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51888,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:38.31109282Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"6aa7d5b9-4497-4bc6-8588-2265a55a8146","remote_addr":"[fd00:1122:3344:101::b]:61182","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:23:56.635332852Z","hostname":"sock","pid":1555,"uri":"/09996097-89b8-4c39-a1ef-fd8405b108d6","method":"GET","req_id":"a600310a-e700-4819-b24a-e29271c14a3f","remote_addr":"[fd00:1122:3344:101::d]:47427","local_addr":"[fd00:1122:3344:101::1]:40617","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":225,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:04.857467398Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33682"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:04.857575949Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"71e76327-444a-4693-8653-27320d89e784","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33682","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":73,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:05.944425273Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59298"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:05.944796356Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"0a74c299-c326-4c71-b4d6-36006a92af54","remote_addr":"[fd00:1122:3344:101::c]:59298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":346,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:06.101044731Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:43950"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:06.147717665Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ddd78603-15bc-4dbd-9b38-7c6779f096ac","remote_addr":"[fd00:1122:3344:101::c]:43950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46616,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:06.147997948Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"6ad7bab3-fdb9-4733-ac09-f59dbb814b67","remote_addr":"[fd00:1122:3344:101::c]:43950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:08.158569521Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:37982"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:08.158752593Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"fb2b70ad-b3df-4049-8c08-5c281f2c38ff","remote_addr":"[fd00:1122:3344:101::b]:37982","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":174,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:08.166875295Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50409"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:08.167009396Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"6674f89a-5a13-4f3e-9a86-63687f529cd3","remote_addr":"[fd00:1122:3344:101::a]:50409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:08.271076722Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:08.318652025Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"774cb1cf-4f95-455a-bc49-77a16db63164","remote_addr":"[fd00:1122:3344:101::a]:60493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47511,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:08.318925697Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"c64ae07d-3262-4c3c-be9b-17efda282ebb","remote_addr":"[fd00:1122:3344:101::a]:60493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:08.595847506Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56679"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:08.64448968Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5a7afb10-f3d7-41be-a848-b47a673d72ff","remote_addr":"[fd00:1122:3344:101::b]:56679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48561,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:08.644782003Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2bc39518-89ba-4eaf-ab84-9db3775d2f9a","remote_addr":"[fd00:1122:3344:101::b]:56679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:26.678909071Z","hostname":"sock","pid":1555,"uri":"/09996097-89b8-4c39-a1ef-fd8405b108d6","method":"GET","req_id":"91854b77-fb69-4fb7-9fba-bd7835886cc9","remote_addr":"[fd00:1122:3344:101::d]:47427","local_addr":"[fd00:1122:3344:101::1]:40617","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":833,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:34.889932764Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50183"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:34.890064876Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"2463417c-5ccc-4ecb-8873-a85e741907f8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50183","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:35.653254637Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:44144"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:35.653506289Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"2faccf09-4c36-44fb-80a8-e86b2468ebe6","remote_addr":"[fd00:1122:3344:101::c]:44144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":216,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:35.851601198Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:40289"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:35.899263482Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"96e82311-bd84-47de-b5a9-d0497f092eb9","remote_addr":"[fd00:1122:3344:101::c]:40289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47606,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:35.899526435Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"46244cc8-c79c-4060-a9ac-67a231e34d54","remote_addr":"[fd00:1122:3344:101::c]:40289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:37.901855524Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:49367"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:37.902044616Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"7327efec-57d7-49e5-b58e-9646463d355b","remote_addr":"[fd00:1122:3344:101::a]:49367","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":172,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:37.995056709Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47403"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:38.045525211Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"182e7936-3c70-4703-82c4-e33faf361b6a","remote_addr":"[fd00:1122:3344:101::a]:47403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50396,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:38.045845905Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1ca170f1-508e-49fe-9856-376dd8c3ca51","remote_addr":"[fd00:1122:3344:101::a]:47403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:38.094770671Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:62345"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:38.095085524Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"08d40a21-5a25-4136-9da5-dc31173f7dca","remote_addr":"[fd00:1122:3344:101::b]:62345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":247,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:38.223138083Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:46028"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:38.272391883Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"fce346ab-a72a-4c06-b3af-982371a66d72","remote_addr":"[fd00:1122:3344:101::b]:46028","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49185,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:38.272712336Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"20b75640-00aa-452b-a9b8-1639878de6cb","remote_addr":"[fd00:1122:3344:101::b]:46028","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":136,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:24:56.679474524Z","hostname":"sock","pid":1555,"uri":"/09996097-89b8-4c39-a1ef-fd8405b108d6","method":"GET","req_id":"22f764d3-ad9a-45f8-babf-bb8cfffcff2e","remote_addr":"[fd00:1122:3344:101::d]:47427","local_addr":"[fd00:1122:3344:101::1]:40617","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":548,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:04.922155719Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54341"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:04.9223032Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"0c71e6df-d817-4a8b-ad38-4da4917a7899","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54341","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:05.755104517Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:36323"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:05.755335249Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"68d8eff0-4e1f-4962-ab52-073f4e28ec46","remote_addr":"[fd00:1122:3344:101::c]:36323","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":211,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:05.996325824Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55912"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:05.996539046Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:05.996586726Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:05.996618047Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:05.996624737Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:05.996637777Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:05.996662457Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:05.996694057Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:05.996709848Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7adf6a1b-8ff9-48e3-895b-3b074e752841","remote_addr":"[fd00:1122:3344:101::c]:55912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":340,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:06.065845149Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:37390"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:06.116521193Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"eda04f36-586f-4d3f-bdcc-f2b3f2a1fc45","remote_addr":"[fd00:1122:3344:101::c]:37390","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50639,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:06.116768235Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f39d99e6-e3e6-4fc9-a224-324c29fb8d99","remote_addr":"[fd00:1122:3344:101::c]:37390","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.050435059Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:61252"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.051446569Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.051495099Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.05152843Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.05154254Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.05154688Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.05155534Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.05156336Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.0515872Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"67efdc8a-29ae-4af7-820b-7afa5465998a","remote_addr":"[fd00:1122:3344:101::a]:61252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1100,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.070675724Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64720"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.070856966Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"0416c856-2316-46d1-8b85-6c7e8ea36135","remote_addr":"[fd00:1122:3344:101::a]:64720","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.267924995Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64121"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.319975693Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e74d7952-25fe-458f-b693-6fc7cf065948","remote_addr":"[fd00:1122:3344:101::a]:64121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51990,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.320088324Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:46585"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.320254445Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"ec293b59-3652-4315-a73c-bd56d461945c","remote_addr":"[fd00:1122:3344:101::b]:46585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":137,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.320431197Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"30a00157-25db-4f36-99fe-48d74be4d669","remote_addr":"[fd00:1122:3344:101::a]:64121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.395567529Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:65112"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.396506289Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3000bf0a-c93f-4744-9659-94404143543f, kind: Service { id: 18077c32-d18c-41ec-bf4f-0e30a06c25cb }, name: Name(\"external-dns-18077c32-d18c-41ec-bf4f-0e30a06c25cb\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 132])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 342cb883-a21c-428b-88d4-e85583f3ca8a, kind: Service { id: cadebd1f-a7c1-4f50-9b00-0055fd0e092a }, name: Name(\"external-dns-cadebd1f-a7c1-4f50-9b00-0055fd0e092a\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 9])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2af9e55a-1eb2-46b8-94b5-b209ad2d36af, kind: Service { id: 267680bb-9aa5-4914-b636-f4328bb9d704 }, name: Name(\"nexus-267680bb-9aa5-4914-b636-f4328bb9d704\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 137, 255])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d31f6d94-cccc-4eab-a49d-731b401e4364, kind: Service { id: cd88d589-b585-424a-9bd6-4d9a048238bc }, name: Name(\"nexus-cd88d589-b585-424a-9bd6-4d9a048238bc\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 197, 92])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e26df023-8dd3-446d-89ab-e987504940fe, kind: Service { id: c34d2d1f-8939-4dc7-829a-e8d77861a9e3 }, name: Name(\"nexus-c34d2d1f-8939-4dc7-829a-e8d77861a9e3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 232])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.39658691Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.3966351Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.396658991Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.396684471Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.396703981Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.396729251Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.396754571Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a9d8a0bc-62e7-426d-ace1-2d90cbe0856f","remote_addr":"[fd00:1122:3344:101::b]:65112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1106,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.503625445Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:55547"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.550335019Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"611ec5da-6e84-49d8-a0aa-a2407ab5cd4b","remote_addr":"[fd00:1122:3344:101::b]:55547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46586,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:08.55140647Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a297341e-4c33-4318-95bc-31bf7ea2ed0a","remote_addr":"[fd00:1122:3344:101::b]:55547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":651,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:26.679642382Z","hostname":"sock","pid":1555,"uri":"/09996097-89b8-4c39-a1ef-fd8405b108d6","method":"GET","req_id":"792017e3-b61b-47b4-adb0-217800fef0e5","remote_addr":"[fd00:1122:3344:101::d]:47427","local_addr":"[fd00:1122:3344:101::1]:40617","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":650,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:34.953399742Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46619"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:34.953507604Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"8973a510-b20f-4691-8fb1-0f41b8b68510","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46619","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":69,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:35.65802663Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:35979"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:35.658250652Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"61e72115-5a09-4db1-b014-88cd990b0397","remote_addr":"[fd00:1122:3344:101::c]:35979","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":207,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:35.787635164Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:46842"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:35.838692942Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a4225b83-826b-495f-8770-c1cd8e12e075","remote_addr":"[fd00:1122:3344:101::c]:46842","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50999,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:35.838954895Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"27414fad-0e99-4c87-91f6-be300527169c","remote_addr":"[fd00:1122:3344:101::c]:46842","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:37.874310039Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53347"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:37.874544832Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"9ac121a1-dfbe-44a4-8aae-66232e2c36f1","remote_addr":"[fd00:1122:3344:101::a]:53347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":193,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:38.017433811Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53230"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:38.064475958Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a647b9a3-b3dd-46b6-8eca-ac684a1eae23","remote_addr":"[fd00:1122:3344:101::a]:53230","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46978,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:38.064733481Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1becbb7d-42b6-4f44-9e2a-516bd0122c5b","remote_addr":"[fd00:1122:3344:101::a]:53230","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:38.165532393Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:65470"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:38.165732195Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"5d12ca0e-56ba-483e-8a28-f8c587f93dce","remote_addr":"[fd00:1122:3344:101::b]:65470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":180,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:38.262937631Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:33075"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:38.312641606Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"99b65a8b-ccde-45db-a8b4-3e9f48e7e174","remote_addr":"[fd00:1122:3344:101::b]:33075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49636,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:38.312929278Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1d2dfd1b-12d6-4c55-9a2c-43dfffd8c63a","remote_addr":"[fd00:1122:3344:101::b]:33075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":107,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:25:56.680039704Z","hostname":"sock","pid":1555,"uri":"/09996097-89b8-4c39-a1ef-fd8405b108d6","method":"GET","req_id":"c9a2cf92-e836-460c-8759-f77cca7422cc","remote_addr":"[fd00:1122:3344:101::d]:47427","local_addr":"[fd00:1122:3344:101::1]:40617","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":582,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:04.984739558Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56475"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:04.984901109Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"07011d94-c344-471b-8e85-7ffdbe4b1731","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:56475","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:05.874711619Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:48222"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:05.874930874Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"e0466f6c-c1b0-4dc1-841a-c68a0dd2cced","remote_addr":"[fd00:1122:3344:101::c]:48222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":206,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:06.098888839Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59504"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:06.148178829Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"afdc4f56-dda2-4e96-a20f-de97ca2ac2a9","remote_addr":"[fd00:1122:3344:101::c]:59504","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49214,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:06.149179879Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"934abd74-7bd5-4b9b-9e7d-55f96a35e1c5","remote_addr":"[fd00:1122:3344:101::c]:59504","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":823,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:07.960000265Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58796"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:07.960198667Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"72f5c8e6-cfbb-454e-ad3c-5d3fd8bc1e09","remote_addr":"[fd00:1122:3344:101::a]:58796","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:08.146873711Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:55487"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:08.147021822Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"b6b2a5cf-8eb3-4161-a158-cbbf3ceb2db6","remote_addr":"[fd00:1122:3344:101::b]:55487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:08.230288807Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47003"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:08.27883965Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"87e91092-f741-4775-9229-8bd87b134bf2","remote_addr":"[fd00:1122:3344:101::a]:47003","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48505,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:08.279101112Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"5dbee004-b51d-41ab-a35b-33e83025eb30","remote_addr":"[fd00:1122:3344:101::a]:47003","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:08.511147076Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:63540"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:08.561287125Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b8bfb869-bb7c-401c-94f4-477b67ea76e4","remote_addr":"[fd00:1122:3344:101::b]:63540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50038,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:08.562064452Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f71378ad-167f-4a02-a903-a0dd66ab47c1","remote_addr":"[fd00:1122:3344:101::b]:63540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":603,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:26.680250273Z","hostname":"sock","pid":1555,"uri":"/09996097-89b8-4c39-a1ef-fd8405b108d6","method":"GET","req_id":"8dbc1019-38ef-4a47-b865-88b9ec93e483","remote_addr":"[fd00:1122:3344:101::d]:47427","local_addr":"[fd00:1122:3344:101::1]:40617","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"09996097-89b8-4c39-a1ef-fd8405b108d6","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":539,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:27.238060781Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:27.23990868Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:27.282538613Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999562s","next_cleanup":"Instant { tv_sec: 1909, tv_nsec: 326172569 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.380264577Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44242"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.380475409Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"94eedb27-8d68-4190-b5f6-2fef2209b627","remote_addr":"[fd00:1122:3344:101::a]:44242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":193,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.384582581Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:631","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 243, 194, 203])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(10745114) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.384611781Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"PUT","req_id":"c17c909d-92cf-476b-906f-6630b32441c7","remote_addr":"[fd00:1122:3344:101::a]:44242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":136,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.569968461Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62571"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.576148274Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:548","metadata":"InstanceMetadata { silo_id: cf6eba96-5322-48f0-8d15-e33744d4b47a, project_id: 4a0591cd-9dcd-4380-8401-385e9f6f6d59 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-17T21:26:28.076791Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-17T21:26:28.136751207Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 6fbee68c-4473-4004-849b-72ee591e1c86, kind: Instance { id: d3dc701c-f09f-4c7d-babc-659cf6b9cfbf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 194, 203])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10745114), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6fbee68c-4473-4004-849b-72ee591e1c86, kind: Instance { id: d3dc701c-f09f-4c7d-babc-659cf6b9cfbf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 194, 203])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10745114), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6fbee68c-4473-4004-849b-72ee591e1c86, kind: Instance { id: d3dc701c-f09f-4c7d-babc-659cf6b9cfbf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 194, 203])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10745114), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(10745114))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6fbee68c-4473-4004-849b-72ee591e1c86, kind: Instance { id: d3dc701c-f09f-4c7d-babc-659cf6b9cfbf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 194, 203])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10745114), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-5a021cc0a3e\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: a4d088b6-685e-4c7b-95e0-98048b746ffb, read_only_parent: Some(Volume { block_size: 512, id: a7ee24b4-148a-42c9-b310-4c35bd892d0c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: c34f3d88-60bd-4606-86b5-93fdefd39b40, key: Some(\"0Mr25qJFUAev1UWZYPxBiEs0bQ/lTEwZf3bkizhcOcU=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a4d088b6-685e-4c7b-95e0-98048b746ffb, key: Some(\"VMHgpwKVmISdHo4/nmGtogcRShsnZNFX6h080U4xvHg=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19003\", \"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::19]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAA1TqvRWNFYAABOq9FYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAANU6r0VjRWAAATqvRWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJkM2RjNzAxYy1mMDlmLTRjN2QtYmFiYy02NTljZjZiOWNmYmYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRlFSRVpkbTdRQzh5RVFiVnJINnRCZm83Ui83eTYwZ00yVGdjcHJ1TXI4bCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"2fdb3551-0640-4f4b-bbf2-a42cb064f493","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.576393466Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:582","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf (instance)"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.576508587Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance.rs:942","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 6fbee68c-4473-4004-849b-72ee591e1c86, kind: Instance { id: d3dc701c-f09f-4c7d-babc-659cf6b9cfbf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 194, 203])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10745114), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6fbee68c-4473-4004-849b-72ee591e1c86, kind: Instance { id: d3dc701c-f09f-4c7d-babc-659cf6b9cfbf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 194, 203])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10745114), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6fbee68c-4473-4004-849b-72ee591e1c86, kind: Instance { id: d3dc701c-f09f-4c7d-babc-659cf6b9cfbf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 194, 203])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10745114), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(10745114))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6fbee68c-4473-4004-849b-72ee591e1c86, kind: Instance { id: d3dc701c-f09f-4c7d-babc-659cf6b9cfbf }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 194, 203])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(10745114), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-5a021cc0a3e\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: a4d088b6-685e-4c7b-95e0-98048b746ffb, read_only_parent: Some(Volume { block_size: 512, id: a7ee24b4-148a-42c9-b310-4c35bd892d0c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: c34f3d88-60bd-4606-86b5-93fdefd39b40, key: Some(\"0Mr25qJFUAev1UWZYPxBiEs0bQ/lTEwZf3bkizhcOcU=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a4d088b6-685e-4c7b-95e0-98048b746ffb, key: Some(\"VMHgpwKVmISdHo4/nmGtogcRShsnZNFX6h080U4xvHg=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19003\", \"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::19]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAA1TqvRWNFYAABOq9FYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAANU6r0VjRWAAATqvRWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJkM2RjNzAxYy1mMDlmLTRjN2QtYmFiYy02NTljZjZiOWNmYmYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRlFSRVpkbTdRQzh5RVFiVnJINnRCZm83Ui83eTYwZ00yVGdjcHJ1TXI4bCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-17T21:26:28.136751207Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-17T21:26:28.076791Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"2fdb3551-0640-4f4b-bbf2-a42cb064f493","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.576644989Z","hostname":"sock","pid":1555,"uri":"/instances/d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","method":"PUT","req_id":"3307b3a5-c8ca-4e2e-8e79-4c4995b75b32","remote_addr":"[fd00:1122:3344:101::a]:62571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":6645,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.71484493Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48890"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.71874504Z","hostname":"sock","pid":1555,"uri":"/instances/d3dc701c-f09f-4c7d-babc-659cf6b9cfbf/state","method":"PUT","req_id":"28045834-78e1-4266-9490-868cf9f52f7d","remote_addr":"[fd00:1122:3344:101::a]:48890","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3822,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.742177077Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 243, 194, 203]), slot: 0, vni: Vni { inner: 10745114 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.742265548Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.775567996Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:28.796738881Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:30.044954521Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:30.045082612Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\") failed: Property not found. retry in 71.617946ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:30.232577604Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\") failed: Property not found. retry in 129.400999ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:30.324390996Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\") failed: Property not found. retry in 145.89295ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:30.472450378Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\") failed: Property not found. retry in 230.248013ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:30.632623672Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\") failed: Property not found. retry in 1.040314914s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:30.876185862Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\") failed: Property not found. retry in 887.507429ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:31.927167302Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\") failed: Property not found. retry in 678.979677ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:32.8241313Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\") failed: Property not found. retry in 1.496228106s","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:33.512518383Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.025124825Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46426"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.025251756Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"20d09a30-0478-49bf-90ee-6f966d9df739","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46426","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":107,"response_code":"200"} {"msg":"Started propolis in zone: oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.037529581Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:1427"} {"msg":"wait for service Some(\"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\") failed: Property not found. retry in 65.425229ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:35.04628425Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\") failed: Property not found. retry in 113.829082ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:35.121563913Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\") failed: Property not found. retry in 221.537ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:35.245779983Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\") failed: Property not found. retry in 268.754971ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:35.476276221Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.690111189Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.6921345Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"cf041d80-5207-4a53-88b5-baedab0459f3","remote_addr":"[fd00:1122:3344:101::c]:57451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":674,"response_code":"200"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.756715655Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:1436"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.790460667Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:50004"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.792638149Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:1450"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAA1TqvRWNFYAABOq9FYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAANU6r0VjRWAAATqvRWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJkM2RjNzAxYy1mMDlmLTRjN2QtYmFiYy02NTljZjZiOWNmYmYiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRlFSRVpkbTdRQzh5RVFiVnJINnRCZm83Ui83eTYwZ00yVGdjcHJ1TXI4bCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-5a021cc0a3e\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: a4d088b6-685e-4c7b-95e0-98048b746ffb, read_only_parent: Some(Volume { block_size: 512, id: a7ee24b4-148a-42c9-b310-4c35bd892d0c, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: c34f3d88-60bd-4606-86b5-93fdefd39b40, key: Some(\"0Mr25qJFUAev1UWZYPxBiEs0bQ/lTEwZf3bkizhcOcU=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a4d088b6-685e-4c7b-95e0-98048b746ffb, key: Some(\"VMHgpwKVmISdHo4/nmGtogcRShsnZNFX6h080U4xvHg=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19003\", \"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::19]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: d3dc701c-f09f-4c7d-babc-659cf6b9cfbf, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 4a0591cd-9dcd-4380-8401-385e9f6f6d59, silo_id: cf6eba96-5322-48f0-8d15-e33744d4b47a }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.795110005Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:689"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.850780799Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.905226872Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"fbf33ecf-3853-40a2-b233-c4cd4c19f9b3","remote_addr":"[fd00:1122:3344:101::c]:58483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":54385,"response_code":"204"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.905489354Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:691"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.906392983Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"80f08f56-6622-4223-989c-aebb78643104","remote_addr":"[fd00:1122:3344:101::c]:58483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":723,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-06-17T21:26:35.907050170Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.90708675Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.907102791Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-17T21:26:35.907050170Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-17T21:26:28.136751207Z }","propolis_id":"2fdb3551-0640-4f4b-bbf2-a42cb064f493"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.907139011Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-17T21:26:28.136751207Z }, propolis_id: 2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-17T21:26:35.907050170Z }, migration_state: None }","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.984222043Z","hostname":"sock","pid":1555,"uri":"/instances/d3dc701c-f09f-4c7d-babc-659cf6b9cfbf/state","method":"GET","req_id":"b2a4ee35-7064-41ac-bb05-5e4d91d887e8","remote_addr":"[fd00:1122:3344:101::c]:50004","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":193710,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-06-17T21:26:35.984442555Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.984458765Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.984472146Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-17T21:26:35.984442555Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-17T21:26:28.136751207Z }","propolis_id":"2fdb3551-0640-4f4b-bbf2-a42cb064f493"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:35.984500096Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-17T21:26:28.136751207Z }, propolis_id: 2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-17T21:26:35.984442555Z }, migration_state: None }","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:37.89860121Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:37.899423658Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"77faeda5-45fe-425e-8e15-9b26a31311de","remote_addr":"[fd00:1122:3344:101::a]:58650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":810,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:37.927556204Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34251"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:37.927696165Z","hostname":"sock","pid":1555,"uri":"/instances/d3dc701c-f09f-4c7d-babc-659cf6b9cfbf/state","method":"GET","req_id":"21b27a23-7568-4bfe-bc93-a51c47b81a96","remote_addr":"[fd00:1122:3344:101::a]:34251","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:38.054641313Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62341"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:38.104036594Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"76ccdb5d-081e-48ed-9164-0edeb947de45","remote_addr":"[fd00:1122:3344:101::a]:62341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49288,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:38.104185815Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:33288"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:38.104455408Z","hostname":"sock","pid":1555,"uri":"/instances/d3dc701c-f09f-4c7d-babc-659cf6b9cfbf/state","method":"GET","req_id":"8f37af98-2edb-4f97-a533-8bf53a023f00","remote_addr":"[fd00:1122:3344:101::b]:33288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":208,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:38.10458707Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e4910b98-f44c-4d44-96f5-5509df79ef95","remote_addr":"[fd00:1122:3344:101::a]:62341","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":244,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:38.112394649Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:53137"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:38.112654752Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"e2de3aa0-5d54-4dc9-9dca-4d7a22e65a74","remote_addr":"[fd00:1122:3344:101::b]:53137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":189,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:38.371595718Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:34640"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:38.417667535Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"efabd18e-7228-4a39-8daa-3adbe95b1b71","remote_addr":"[fd00:1122:3344:101::b]:34640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46013,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:38.418028049Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"fd95806e-2207-4190-8efa-da1787f8ebba","remote_addr":"[fd00:1122:3344:101::b]:34640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":104,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:40.716980477Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:46.121509954Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34842"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:46.122218212Z","hostname":"sock","pid":1555,"uri":"/instances/d3dc701c-f09f-4c7d-babc-659cf6b9cfbf/state","method":"PUT","req_id":"0540be38-2238-4f05-8cb9-bf3a33887d37","remote_addr":"[fd00:1122:3344:101::a]:34842","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":655,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-06-17T21:26:46.122330493Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:46.122355823Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:46.122378443Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-17T21:26:46.122330493Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-17T21:26:28.136751207Z }","propolis_id":"2fdb3551-0640-4f4b-bbf2-a42cb064f493"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:46.122399553Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-17T21:26:28.136751207Z }, propolis_id: 2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-17T21:26:46.122330493Z }, migration_state: None }","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-06-17T21:26:46.173259169Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:46.17328551Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:46.17331287Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-06-17T21:26:46.173259169Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-17T21:26:46.173259169Z }","propolis_id":"2fdb3551-0640-4f4b-bbf2-a42cb064f493"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:46.17336425Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:610","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:46.17337103Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:46.173613333Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:46.173687264Z","hostname":"sock","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/506b1cae-3bee-435f-ba3e-326026eb5f18/debug/bundle/zone\", \"/pool/int/b8f4ab32-a0e3-4946-a6a4-a7402b5b0422/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/ffe46556-d7d9-435b-85f0-4376aa51f547/crypt/debug\", \"/pool/ext/2abfdf07-7cd1-4355-89f0-e7d8ec686d7c/crypt/debug\", \"/pool/ext/e4f51e96-8b35-4542-822a-22c6778e005f/crypt/debug\", \"/pool/ext/d3e797bc-6c3a-4d22-8dd0-0eb78cd24c98/crypt/debug\", \"/pool/ext/38ec74ae-3290-4738-828b-35608cdf0b99/crypt/debug\", \"/pool/ext/cea92678-6b55-4860-a3b6-57f49d0a71a4/crypt/debug\", \"/pool/ext/ab647d9b-1f8b-4c75-927b-5b8bc52c8141/crypt/debug\", \"/pool/ext/fa78ab30-cb65-4bf2-8677-79fb8f81cd6b/crypt/debug\", \"/pool/ext/708ddef6-4ed2-4ac7-bfb0-97e7a7008313/crypt/debug\"] }","zone_name":"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:46.816085149Z","hostname":"sock","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493\", bundle_id: 287de308-7ff8-4ab7-9b5c-56ee4d3496f7 }, time_created: 2024-06-17T21:26:46.173933756Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_2fdb3551-0640-4f4b-bbf2-a42cb064f493","v":0,"name":"SledAgent","level":40,"time":"2024-06-17T21:26:46.816219811Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:784"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:48.568636056Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:48.593999173Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-17T21:26:46.173259169Z }, propolis_id: 2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-17T21:26:48.593932942Z }, migration_state: None }","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:48.669917633Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:48.670124865Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"1b46f651-b370-4a4f-ba03-585d40f21ff1","remote_addr":"[fd00:1122:3344:101::c]:54188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":175,"response_code":"200"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:48.892277378Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-17T21:26:46.173259169Z }, propolis_id: 2fdb3551-0640-4f4b-bbf2-a42cb064f493 (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-17T21:26:48.593932942Z }, migration_state: None }","instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:48.895673672Z","hostname":"sock","pid":1555,"instance_id":"d3dc701c-f09f-4c7d-babc-659cf6b9cfbf","component":"InstanceManager","file":"sled-agent/src/instance.rs:727"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:49.104804804Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62638"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-17T21:26:49.104981835Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f40966a0-f9b9-4577-b609-ed907b8638fe","remote_addr":"[fd00:1122:3344:101::a]:62638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":127,"response_code":"200"}