[ Apr 29 04:08:15 Enabled. ] [ Apr 29 04:08:15 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Apr 29 04:08:15 Method "start" exited with status 0. ] [ Apr 29 04:08:15 Rereading configuration. ] [ Apr 29 04:08:15 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:15.929051768Z","hostname":"sock","pid":1590,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:15.929222702Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:15.982430093Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:15.982472894Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.097219727Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.106576089Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.115290347Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.717677561Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.860158244Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.860195744Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.860293726Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.860428029Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.860434579Z","hostname":"sock","pid":1590,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:291"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.86047068Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.86047866Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.86340498Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:16.863419221Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.863427081Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.863431281Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.863457381Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.863523363Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.863571694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.863587614Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.863611794Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.875375596Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.875411996Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:16.999242405Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.235019871Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.235068892Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.235080572Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.235098852Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.235108533Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.235115893Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.235142833Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.243861512Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.243893933Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.348997648Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.589910949Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.58995403Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590015941Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590032672Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590048872Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590091223Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590110933Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590115764Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590143074Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590155704Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590183055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590187445Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590193895Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590198195Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590215286Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590219486Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590226046Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590232686Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590237136Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590243036Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590260096Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590273387Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590287177Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590293187Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590312097Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590317197Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590323088Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590328168Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590347668Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590354888Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590368519Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590376809Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590399399Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590406209Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.59041043Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.59041886Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.59044907Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.59046104Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590466861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590473751Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590482011Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"No ledger in /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590530392Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590547812Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590554982Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590566113Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590576943Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590595123Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590601994Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.590610674Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.602084769Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.60215084Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999921238s","next_cleanup":"Instant { tv_sec: 1502, tv_nsec: 601953366 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.60216524Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.602203281Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.602261673Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605104711Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605460518Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.6055276Z","hostname":"sock","pid":1590,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605770534Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:979"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605779075Z","hostname":"sock","pid":1590,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605826866Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605869556Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605896447Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605900987Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605925618Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605938588Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605952858Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605958578Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1080"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605962289Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605966478Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.605992859Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.606013219Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.606341456Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.606579651Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.606594241Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1113"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.606640192Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.606650552Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.606797695Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.606916938Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.606961249Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 94bd4353-ea99-483d-acc9-6db1c4f75870, rack_id: 57b47f3e-5d38-4c51-9327-55cd3ad29f29, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.607076221Z","hostname":"sock","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:42519","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 94bd4353-ea99-483d-acc9-6db1c4f75870, rack_id: 57b47f3e-5d38-4c51-9327-55cd3ad29f29, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.607185754Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.607190904Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.607226474Z","hostname":"sock","pid":1590,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.607230815Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.607250665Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.607254625Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.607259415Z","hostname":"sock","pid":1590,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"No ledger in /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.607264795Z","hostname":"sock","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.607292306Z","hostname":"sock","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.607299466Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.607302856Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.609984301Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.618068917Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.62845892Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.639531647Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.649593303Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.658885774Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.668687005Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.677419664Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.686421928Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.695379742Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.695427503Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.695434563Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.695439513Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.695485765Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.695492094Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.715638058Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.733046625Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.744575891Z","hostname":"sock","pid":1590,"sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:352"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.744614922Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.744620762Z","hostname":"sock","pid":1590,"sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:364"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.744631612Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 108.043742ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.747545012Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eae96cea-7ca4-4ed3-97c8-9dd3af34d612\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:08:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eae96cea-7ca4-4ed3-97c8-9dd3af34d612\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_50faa4b2-c048-4a8a-9779-53ba148e2f90/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.759303093Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_50faa4b2-c048-4a8a-9779-53ba148e2f90/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.787263117Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_50faa4b2-c048-4a8a-9779-53ba148e2f90/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.787307538Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.792532955Z","hostname":"sock","pid":1590,"sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:379"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.792546455Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_50faa4b2-c048-4a8a-9779-53ba148e2f90/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.792551905Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.856712031Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 107.527462ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.857322453Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"df8b21f5-4053-4da6-9495-53588ae03cf8\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:08:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"df8b21f5-4053-4da6-9495-53588ae03cf8\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.873455984Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.966581673Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 191.524968ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:17.967283528Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6bbe4a07-1bf5-4eb5-aea3-f24b90e82733\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:08:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6bbe4a07-1bf5-4eb5-aea3-f24b90e82733\" }"} {"msg":"Mounting oxi_50faa4b2-c048-4a8a-9779-53ba148e2f90/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.092123358Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.104295028Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.104537293Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.104588474Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.104632384Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.160391038Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 122.803211ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.161020621Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6a3794e5-b3b2-4e00-8a22-cd0c5c5d91c9\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:08:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6a3794e5-b3b2-4e00-8a22-cd0c5c5d91c9\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.285068385Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 140.184794ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.285419502Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"164f7f9f-7830-403f-8f74-725e8bdd8dfb\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:08:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"164f7f9f-7830-403f-8f74-725e8bdd8dfb\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.4267182Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 281.049626ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.427550017Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0874aa63-d90b-47c5-be7d-e709dbdd1e46\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:08:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0874aa63-d90b-47c5-be7d-e709dbdd1e46\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.709167543Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 196.823727ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.709894008Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4f7b30bc-3ebb-4fe6-8d90-09d67281ebf0\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:08:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4f7b30bc-3ebb-4fe6-8d90-09d67281ebf0\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.819468585Z","hostname":"sock","pid":1590,"sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.820102838Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:195","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:195"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.820127498Z","hostname":"sock","pid":1590,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.82018422Z","hostname":"sock","pid":1590,"file":"oximeter/producer/src/lib.rs:284","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:284","interval":"30s","address":"[fd00:1122:3344:101::1]:48068","producer_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","route":"//94bd4353-ea99-483d-acc9-6db1c4f75870"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:18.826623572Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"213.434859ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.869078342Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.869112793Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.869189095Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.869198615Z","hostname":"sock","pid":1590,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.869862658Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:195"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.869885969Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.869913319Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.8699243Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.870160945Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.870317708Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.870367549Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100 }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.870379029Z","hostname":"sock","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:42519","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:18.870397449Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.082648222Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.084118022Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.084168283Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.084426428Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.08452459Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.084566561Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.084576701Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.084593161Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:19.085444469Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"629.359524ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:19.088951661Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.116779952Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63481"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.116998126Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.117110648Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"845576a9-9e5b-4b89-bcf0-f880c258348b","remote_addr":"[fd00:1122:3344:101::1]:63481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":207,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.146885009Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:62597"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.147015682Z","hostname":"sock","pid":1590,"uri":"/sled-role","method":"GET","req_id":"323a1653-ab8a-48dd-a627-ce67d5472169","remote_addr":"[fd00:1122:3344:101::1]:62597","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":51,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.148629855Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.148666536Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.148953432Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.149145525Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: f741ba07-205a-47e0-9fc1-4ff20b5dea71, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool) }, OmicronPhysicalDiskConfig { id: b69fcc99-0404-4043-a682-ea2760839f5a, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool) }, OmicronPhysicalDiskConfig { id: 415ad3fb-eb08-468c-b620-37bb4856ec23, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool) }, OmicronPhysicalDiskConfig { id: a8e61296-f9ac-4ba1-934a-670ed1dcd557, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool) }, OmicronPhysicalDiskConfig { id: 9cb7dd83-a5a4-44c8-abfe-8425d58e59bc, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool) }, OmicronPhysicalDiskConfig { id: 115e9f78-9dee-47f1-9e8c-796c25e41352, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool) }, OmicronPhysicalDiskConfig { id: 78a1bee3-44e5-4514-aafa-c64cb4e663a7, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool) }, OmicronPhysicalDiskConfig { id: fe49bac8-9d08-421a-a008-e7f3dae4c6a8, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool) }, OmicronPhysicalDiskConfig { id: a567c2b9-8d49-4e24-b820-b631b77d2667, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.17908763Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:346"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.179179052Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:44934"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: f741ba07-205a-47e0-9fc1-4ff20b5dea71, pool_id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: b69fcc99-0404-4043-a682-ea2760839f5a, pool_id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: 415ad3fb-eb08-468c-b620-37bb4856ec23, pool_id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: a8e61296-f9ac-4ba1-934a-670ed1dcd557, pool_id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: 9cb7dd83-a5a4-44c8-abfe-8425d58e59bc, pool_id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: 115e9f78-9dee-47f1-9e8c-796c25e41352, pool_id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: 78a1bee3-44e5-4514-aafa-c64cb4e663a7, pool_id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: fe49bac8-9d08-421a-a008-e7f3dae4c6a8, pool_id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: a567c2b9-8d49-4e24-b820-b631b77d2667, pool_id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.179385666Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.179391676Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.179398416Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.179407326Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.179436967Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.179442847Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.179448337Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.179452867Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.304343048Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:19.304385979Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:19.723857282Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"791.636705ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T04:08:19.982057397Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:20.427131845Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:20.645072124Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.496750795s"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:20.650799782Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:20.650886683Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:20.650894304Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:20.650899664Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:20.650908074Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:20.650997446Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:20.651004186Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T04:08:21.086789713Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:21.086840644Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:21.290414009Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:21.518388644Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Automatically destroying dataset: oxp_d1b1e109-515b-482d-930d-ba493cd78ef0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:21.711729509Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T04:08:22.154325036Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 71.898424ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:22.355939521Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 72.38885ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:22.951551046Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 217.49615ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:23.623801612Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:23.626442337Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.62009033s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T04:08:23.626469567Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:23.626495467Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 422.098247ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:24.219984948Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.006868945s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:24.8520032Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T04:08:24.853214395Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_d1b1e109-515b-482d-930d-ba493cd78ef0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:24.864465256Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:25.065944107Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:25.066010749Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:25.066016599Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:25.066023469Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:25.066027799Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:25.272691638Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:25.272728398Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:26.351251597Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:26.57132488Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:26.571391712Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:26.571402042Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:26.571410112Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:26.571432012Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:26.571627586Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:26.571643466Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T04:08:27.037069751Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:27.037107392Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 635.776927ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:27.037179863Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:27.543720542Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7b71d6bd-a486-4799-a3b4-23ca97ac0816/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:27.981672063Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:28.919327943Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T04:08:28.919421755Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.332007636Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:29.333896995Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"8.116202025s"} {"msg":"Looking for unencrypted datasets in oxp_7b71d6bd-a486-4799-a3b4-23ca97ac0816","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.436754154Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.445520024Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.445567005Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.445572815Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.445577665Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.445584105Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.455188952Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.455229583Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.571187251Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.578897309Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.57894171Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.578960421Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.578971281Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.578983251Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.579123214Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.579131504Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.621498313Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a1b82a77-810b-4e64-9e93-3bddcb08ae61/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.629321324Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_a1b82a77-810b-4e64-9e93-3bddcb08ae61","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.770977749Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.779044764Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.779086285Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.779093665Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.779100555Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.779105275Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.789114971Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.789150281Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"24a77df0-4435-4add-9951-0211a2cf4c49 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.902542217Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.910283576Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.910316546Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.910326267Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.910368957Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.910418809Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.910564572Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.910573012Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T04:08:29.921950515Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.953488202Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_24a77df0-4435-4add-9951-0211a2cf4c49/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:29.961385634Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_24a77df0-4435-4add-9951-0211a2cf4c49","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.101838793Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.110011001Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.110065532Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.110074052Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.110082262Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.110091323Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.119987246Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.120013476Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.231770188Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.239703491Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.239774742Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.239784802Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.239822643Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.239852754Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.240025927Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.240037668Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.281656691Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_6686ffc9-9b48-48c1-988b-7aac0a49e5a5/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.289779888Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_6686ffc9-9b48-48c1-988b-7aac0a49e5a5","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.432941734Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.440638952Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.440696803Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.440704743Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.440714433Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.440722873Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.450857441Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.450899232Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.566729288Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.57417807Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.574227591Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.574234611Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.574243962Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.574299483Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.574449046Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.574461266Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.61367622Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_92f3b999-4455-49cb-8ded-aa658c2709d6/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.62143868Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:30.680600353Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Looking for unencrypted datasets in oxp_92f3b999-4455-49cb-8ded-aa658c2709d6","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.764997534Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.773187142Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.773250423Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.773257473Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.773264523Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.773270564Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.782913971Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.782961482Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"feef97bb-ca23-472f-b173-ce352fd7d46c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.893415568Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.901010563Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.901065725Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.901074545Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.901087095Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.901097885Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.901245848Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.901258298Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T04:08:30.924500665Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.94131903Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_feef97bb-ca23-472f-b173-ce352fd7d46c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:30.949882816Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_feef97bb-ca23-472f-b173-ce352fd7d46c","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.095144644Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.10324743Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.103291821Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.103299901Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.103308992Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.103314902Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.112852527Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.112886108Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"9ffa1733-2919-4d64-8165-6d72c0607851 (zpool)"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.190670393Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.229586722Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.237210578Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.237243888Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.237263979Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.23730188Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.23731441Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.237455923Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.237464603Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.27777694Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9ffa1733-2919-4d64-8165-6d72c0607851/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.285300484Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_9ffa1733-2919-4d64-8165-6d72c0607851","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.424372686Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.432072524Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.432112625Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.432129985Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.432135696Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.432140336Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.441546218Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.441575369Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.552203268Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.559720172Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.559759203Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.559767493Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.559801214Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.559812674Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.559976007Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.560003058Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.602678353Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_05c910b1-d5d8-4b62-8b1a-0afed5d62eeb/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.610209217Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_05c910b1-d5d8-4b62-8b1a-0afed5d62eeb","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.751349502Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.75905436Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.759093171Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Writing ledger to /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.759148342Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.759277755Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 50faa4b2-c048-4a8a-9779-53ba148e2f90 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 659cec67-f7a4-40fc-9bb6-e073575516b2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Writing ledger to /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.759506999Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.759805825Z","hostname":"sock","pid":1590,"uri":"/omicron-physical-disks","method":"PUT","req_id":"726bb9ca-9587-4869-8cd7-bfb015abb5ee","remote_addr":"[fd00:1122:3344:101::1]:44934","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":12580557,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.791064127Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.791190569Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51848"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.791428144Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.791438964Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.852590699Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.852634869Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T04:08:31.926585336Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.933681711Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.933697332Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.996272974Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.996351356Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.996369456Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:31.996380517Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:32.042839209Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:32.042898651Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:32.042905261Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:32.096966359Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:32.101041013Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:32.105207128Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:32.117717885Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:32.12234041Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:32.126349772Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:32.680661521Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/94bd4353-ea99-483d-acc9-6db1c4f75870): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-29T04:08:32.928419302Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:34.349926033Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:34.350010445Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:34.350177968Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:34.83443903Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:35.313233589Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:35.313334171Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:35.313346051Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba\") failed: Property not found. retry in 71.78961ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:35.321708673Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:35.321756464Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:35.325601383Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba\") failed: Property not found. retry in 105.374408ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:35.402956589Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f\") failed: Property not found. retry in 52.889881ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:35.449065875Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5\") failed: Property not found. retry in 28.369917ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:35.455052247Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5\") failed: Property not found. retry in 104.967259ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:35.491125347Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f\") failed: Property not found. retry in 69.910079ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:35.510605267Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba\") failed: Property not found. retry in 271.664783ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:35.529043185Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5\") failed: Property not found. retry in 257.299713ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:35.641486561Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f\") failed: Property not found. retry in 227.084881ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:35.679055661Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba\") failed: Property not found. retry in 522.468233ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:35.837778317Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5\") failed: Property not found. retry in 205.056564ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:35.938136585Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f\") failed: Property not found. retry in 269.721103ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:35.947826464Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5\") failed: Property not found. retry in 528.195684ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:36.145959026Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f\") failed: Property not found. retry in 715.083798ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:36.22612927Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba\") failed: Property not found. retry in 742.746202ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:36.368720715Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5\") failed: Property not found. retry in 1.127323927s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:36.684010721Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f\") failed: Property not found. retry in 645.993686ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:36.950307322Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba\") failed: Property not found. retry in 1.086339084s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:37.120350549Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:37.23304235Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:37.233065501Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f\") failed: Property not found. retry in 905.934695ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:37.605791515Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fea2:ba27","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:37.790972943Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:37.796986416Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"} {"msg":"wait for service Some(\"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5\") failed: Property not found. retry in 796.244868ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:37.819530718Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:37.922061081Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:38.170792512Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"} {"msg":"wait for service Some(\"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba\") failed: Property not found. retry in 625.236786ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:38.215029359Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:38.357078212Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"} {"msg":"wait for service Some(\"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f\") failed: Property not found. retry in 776.023711ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:38.520654347Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5\") failed: Property not found. retry in 897.037502ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:38.62369138Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:38.65102815Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:38.843706152Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"} {"msg":"wait for service Some(\"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba\") failed: Property not found. retry in 561.25344ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:38.849537892Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:39.01624973Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:39.220261574Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"} {"msg":"wait for service Some(\"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f\") failed: Property not found. retry in 866.034236ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:39.305421101Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:39.394223802Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:39.404090474Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:39.404124315Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:39.404134005Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba\") failed: Property not found. retry in 1.02433889s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:39.419768786Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5\") failed: Property not found. retry in 1.484201844s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:39.529483466Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:39.955629146Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:39.961735301Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.101892384Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"} {"msg":"wait for service Some(\"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f\") failed: Property not found. retry in 840.857747ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:40.180525017Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.202169381Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.224668182Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.324119482Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.448715137Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:364"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.485301628Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:412"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.519745274Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:589"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.519837726Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:627"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.520346486Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:630","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.520356637Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:434","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612756592Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612851804Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612864814Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612872754Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612879524Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612886795Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612894575Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612901215Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612907875Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612915105Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612922075Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612929215Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612944266Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612949876Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612955236Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612960636Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612967646Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612973206Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612979986Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612985126Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612989917Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612994847Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:40.612999677Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.046446105Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_d8f0884b-aee3-45e7-9763-c04c8a43e0ba"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.046504516Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_08a6de18-4447-48d4-a83c-cd709a350ec5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.046510016Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_b8b4505b-bc72-4995-b8bc-e81e97f16f1f"} {"msg":"Writing ledger to /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.046513577Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.046796482Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.047113169Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"9ab34d0b-c911-4e2c-95fc-cec67918b1b8","remote_addr":"[fd00:1122:3344:101::1]:51848","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":9255837,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.048771403Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"failed to write DNS configuration (will retry in 114.573166ms)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:41.347312846Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:567","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.484615552Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"failed to write DNS configuration (will retry in 96.68507ms)","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:41.513722818Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:567","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:2::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.634064497Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.691524475Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:583"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.691535405Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.723212425Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.725463161Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.725472111Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.753992676Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.754126979Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38571"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.754376214Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.754385024Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.775866205Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.775914266Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.794348524Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.794398805Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.813040497Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.813079128Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.813230621Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.813361344Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.824496002Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.856618711Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.858316256Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.858321256Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.902294058Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 160, 110]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.902364359Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.902950971Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.902957721Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.903282248Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.904721148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.929634608Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:41.948114227Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:43.100854127Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:43.10098285Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\") failed: Property not found. retry in 33.250944ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:43.232869005Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\") failed: Property not found. retry in 95.989072ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:43.273996098Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\") failed: Property not found. retry in 128.13014ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:43.398494202Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\") failed: Property not found. retry in 289.75923ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:43.538580684Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\") failed: Property not found. retry in 927.79702ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:43.837934704Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:43.94159426Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43837"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:43.941775873Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"f0049619-b6df-43af-9918-ffa86653b1f3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43837","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":86,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\") failed: Property not found. retry in 672.30663ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:44.776185045Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\") failed: Property not found. retry in 1.180464982s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:45.457672831Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\") failed: Property not found. retry in 880.813775ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:46.647592464Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\") failed: Property not found. retry in 1.396076073s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:47.537025755Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:48.952651646Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6"} {"msg":"Writing ledger to /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:48.952717548Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:48.953054245Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:48.953348651Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"c74e600a-5ee6-425e-83f9-8acbf2634154","remote_addr":"[fd00:1122:3344:101::1]:38571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":7199159,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:48.955721259Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:624"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:48.984967729Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:48.985107892Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35207"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:48.993388991Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:48.993520044Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"82b9f0c9-aa66-4c1c-ab3d-469ea5d27783","remote_addr":"[fd00:1122:3344:101::1]:35207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8282,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:48.995322921Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:49.129527783Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:49.129683897Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63510"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:53.138201813Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:53.138324726Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"6a13aafa-4192-41c5-b098-fbbfcd7f8cce","remote_addr":"[fd00:1122:3344:101::1]:63510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008533,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:53.141109413Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:53.316081741Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:53.316217954Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63357"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:53.325486354Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:53.325608116Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"32b29e24-c617-4620-a377-8b454b73f8da","remote_addr":"[fd00:1122:3344:101::1]:63357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9215,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:53.326945374Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:53.507769592Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:53.507922546Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:45729"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_b5d37b99-7ecd-4934-8190-c9ab1cf73eb6': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:57.516453442Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:57.516552875Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"4ede1962-85ec-4f10-867c-2016fa52cc8f","remote_addr":"[fd00:1122:3344:101::1]:45729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008502,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:57.520785751Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:57.520974935Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"313.532701ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:57.764656283Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:57.764805816Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:53284"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:57.77379578Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"fa2e3464-d624-4469-b49c-d83cd93ee3e4","remote_addr":"[fd00:1122:3344:101::1]:53284","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":8859,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714363732.923377, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:57.777614739Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:57.777623619Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:57.917374705Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:57.917485127Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:64062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:57.925729336Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"d2b3b542-31b5-4eae-9a1f-9d08a881310f","remote_addr":"[fd00:1122:3344:101::1]:64062","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":8119,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714363732.923377, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:57.927184856Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:57.927190706Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.090162438Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.090303731Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:61398"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.098322735Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"1a71b3b7-c7a3-476a-a0f1-b680487929d2","remote_addr":"[fd00:1122:3344:101::1]:61398","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":7827,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714363732.923377, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.101301167Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.101308987Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.459159456Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.459279168Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:42637"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.468250872Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"ada7d625-53c8-4e2d-88cc-4fefc2da7e4e","remote_addr":"[fd00:1122:3344:101::1]:42637","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":8872,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714363732.923377, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.470441987Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.470451487Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.809527211Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.809642744Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.817995125Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"e1ee8ae9-f9d7-4690-949a-3ffe3b1657ff","remote_addr":"[fd00:1122:3344:101::1]:54483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":8238,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714363732.923377, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.819888854Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.819895434Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.195168462Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.195286855Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:55353"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.198952Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.216207624Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"3192d2bc-1160-4a43-b2cc-462ea451a1a8","remote_addr":"[fd00:1122:3344:101::1]:55353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":20759,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 3093935632, ip_addr: 184.105.182.16, stratum: 4, ref_time: 1714363737.422197, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.21800422Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.218035771Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1192"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.246288491Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.246405743Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:36381"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.246649628Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.246660528Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.268403584Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.268462115Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.285986275Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.286031516Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.302462573Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.302511324Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.361496394Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.361548955Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.422598176Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.422647168Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.48274703Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.482791391Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.542795732Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.542843453Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.601101417Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.601165779Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.601408264Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.601549357Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.633770767Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.633821579Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.633830259Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.633837249Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.633844129Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.633994112Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b284e148-3b9c-437c-bd67-fe1aa8399de8","remote_addr":"[fd00:1122:3344:101::1]:36381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\", TimeNotSynchronized), (\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\", TimeNotSynchronized), (\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\", TimeNotSynchronized), (\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\", TimeNotSynchronized), (\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\", TimeNotSynchronized)]","latency_us":387525,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.634232917Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"70.130253ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b284e148-3b9c-437c-bd67-fe1aa8399de8\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:08:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b284e148-3b9c-437c-bd67-fe1aa8399de8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.705362376Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.705638791Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.705646312Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.722327734Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.722367115Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.73773335Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.73776682Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.754311379Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.75434382Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.770269876Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.770303147Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.78751505Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.787548201Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.804177042Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.804219902Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.820894564Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.820937595Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.838522486Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.838556797Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.83873941Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.838880753Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.869346588Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.869385059Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.869391849Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.869398919Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.86940765Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.869537922Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"1c6c353d-067a-485b-bad9-c197910caf72","remote_addr":"[fd00:1122:3344:101::1]:36381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\", TimeNotSynchronized), (\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\", TimeNotSynchronized), (\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\", TimeNotSynchronized), (\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\", TimeNotSynchronized), (\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\", TimeNotSynchronized)]","latency_us":164013,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:58.869723406Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"94.334068ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1c6c353d-067a-485b-bad9-c197910caf72\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:08:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1c6c353d-067a-485b-bad9-c197910caf72\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.964894068Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.965308566Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.965329297Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.983680813Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:58.983715534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.000712332Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.000754203Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.019899296Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.019938367Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.039792694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.039836105Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.05763033Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.05766976Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.076762392Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.076805693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.094595368Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.094634589Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.112972895Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.113029606Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.11320807Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.113333102Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:59.144142644Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:59.144204085Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:59.144214795Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:59.144224946Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:59.144232886Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.144315377Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"ad17c10a-16e8-4ff5-84ed-88b2a5ed3d3f","remote_addr":"[fd00:1122:3344:101::1]:36381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\", TimeNotSynchronized), (\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\", TimeNotSynchronized), (\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\", TimeNotSynchronized), (\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\", TimeNotSynchronized), (\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\", TimeNotSynchronized)]","latency_us":179221,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:59.144500261Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"199.357282ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ad17c10a-16e8-4ff5-84ed-88b2a5ed3d3f\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:08:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ad17c10a-16e8-4ff5-84ed-88b2a5ed3d3f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.34484704Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.345119306Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.345142416Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.364750548Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.364791589Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.382564274Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.382604695Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.399755786Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.399798767Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.416701824Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.416748495Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.434519389Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.43455859Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.451988198Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.452027558Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.469103398Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.469143749Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.485999615Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.486058976Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.48625069Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.486381623Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:59.51938212Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:59.519432611Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:59.519440951Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:59.519450501Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:59.519472351Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:08:59.519607694Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"048e8412-104f-4025-99a7-c7dc45c177d2","remote_addr":"[fd00:1122:3344:101::1]:36381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\", TimeNotSynchronized), (\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\", TimeNotSynchronized), (\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\", TimeNotSynchronized), (\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\", TimeNotSynchronized), (\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\", TimeNotSynchronized)]","latency_us":174615,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:08:59.519833139Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"197.356319ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"048e8412-104f-4025-99a7-c7dc45c177d2\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:08:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"048e8412-104f-4025-99a7-c7dc45c177d2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.114041479Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.114436157Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.114449847Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.117892618Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.132261032Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.132308864Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.147404513Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.147448134Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.164635396Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.164680337Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.181869239Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.18190529Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.199379518Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.199417529Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.217163753Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.217206934Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.234923907Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.234968758Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.25212687Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.252184831Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.252357845Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.252487897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.329267272Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.329324383Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.329331653Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.329340554Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.329346934Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.41548231Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.419142565Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.422900943Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.42718514Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.431851096Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.436385899Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.440399801Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.444807892Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.449164231Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:00.45351992Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:03.89555959Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:03.895714203Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:03.926387382Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:03.926510585Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:04.005524695Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:04.005668478Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\") failed: Property not found. retry in 41.551052ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:04.052214173Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:04.061920882Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:04.062067735Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 35.049431ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:04.076603633Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\") failed: Property not found. retry in 64.769127ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:04.103408293Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 57.61719ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:04.126614848Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:04.147881675Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:04.148260803Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 57.396501ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:04.253364027Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 166.680807ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:04.439827671Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 136.940366ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:04.451795097Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\") failed: Property not found. retry in 136.76898ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:04.685063481Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 474.153034ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.069891123Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\") failed: Property not found. retry in 69.30827ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.159543181Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 247.547367ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.261116534Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\") failed: Property not found. retry in 475.402906ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.358034202Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 65.344146ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.383834321Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\") failed: Property not found. retry in 124.173426ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.411276874Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 91.693037ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.530287465Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 296.313217ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.549738033Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 1.101657571s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.57005553Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\") failed: Property not found. retry in 152.988946ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.598088125Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 255.594244ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.648988359Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\") failed: Property not found. retry in 509.575935ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.769041011Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\") failed: Property not found. retry in 772.361563ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.850826939Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 475.908953ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.865658703Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 464.463982ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:05.929054983Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\") failed: Property not found. retry in 1.061088087s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:06.298021119Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 854.786273ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:06.357707123Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 419.028486ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:06.444646516Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\") failed: Property not found. retry in 945.075986ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:06.638350169Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 1.211443126s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:06.689038448Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 1.479252201s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:06.880331332Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 855.958174ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:07.222482518Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\") failed: Property not found. retry in 1.419341408s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:07.367427601Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\") failed: Property not found. retry in 945.367793ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:07.592012456Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 1.332698539s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:07.909585209Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 1.002242954s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:08.0880615Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 733.297689ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:08.366741495Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\") failed: Property not found. retry in 566.422358ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:08.54596436Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\") failed: Property not found. retry in 1.499756838s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:08.798711744Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 1.121632007s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:09.101761669Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 1.433571191s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:09.111456978Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\") failed: Property not found. retry in 1.180758572s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:09.127464186Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 1.188852703s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:09.254344038Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 560.347118ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:10.232847875Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\") failed: Property not found. retry in 916.542811ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:10.305012995Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\") failed: Property not found. retry in 1.085525354s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:10.31504245Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 1.334488061s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:10.451895117Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 896.706092ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:10.553405049Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 805.42128ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:10.801269402Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\") failed: Property not found. retry in 906.947543ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:11.233066477Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1\") failed: Property not found. retry in 1.141912404s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:11.409899094Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 904.702211ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:11.458645084Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 1.178361242s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:11.616372868Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 728.58788ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:11.796418331Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:12.056317441Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"537.183948ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d\") failed: Property not found. retry in 1.400913159s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:12.152177086Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 1.322155071s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:12.374127608Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 943.848728ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:12.536047159Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 537.737194ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:12.804506625Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:13.161433284Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61366"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:13.161604307Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"27a48058-1ffe-4b91-a992-444cb28fa06b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61366","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 1.101283722s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:13.352468592Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 710.874497ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:13.491154816Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 1.113826123s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:13.705662805Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 1.032836341s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:14.212699184Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 1.365622591s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:14.464285683Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 725.933936ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:15.29696855Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 849.474178ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:15.30819347Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 986.762944ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:15.841932806Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 638.269184ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:16.033827022Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 1.107483828s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:16.167667036Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 672.536189ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:16.682604457Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 1.126796901s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:16.83931163Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 695.154201ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:17.287893019Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 1.449999192s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:17.368412371Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 674.763255ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:17.977133155Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 1.254510961s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:17.993815247Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 627.620544ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:18.660660723Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 1.080395453s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:18.827091746Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 621.827193ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:19.256717146Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 1.322403957s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:19.296458961Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 1.362570357s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:19.886770098Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 1.232962707s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:19.915504497Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 1.491467537s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:20.629083981Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 1.049015956s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:21.157295333Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 556.268429ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:21.257759973Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98\") failed: Property not found. retry in 704.050712ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:21.824220621Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a\") failed: Property not found. retry in 865.294937ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:22.129626424Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac\") failed: Property not found. retry in 933.322718ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:22.21427814Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:23.168636281Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_b4fbfbe3-741c-42e7-9901-7e3852a09f98"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:23.168705593Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_119d06a5-5604-4ebd-b3fb-bf4627742d2d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:23.168715583Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:23.168724083Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_c369e252-d803-47dd-b8d9-99560ba33cb1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:23.168730843Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_6398723a-debc-46be-9f6c-b8deb81778ac"} {"msg":"Writing ledger to /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:23.168774915Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:23.169176653Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:23.16951918Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"2446bdee-f071-487e-94d7-776a2b0bb99c","remote_addr":"[fd00:1122:3344:101::1]:36381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":23055285,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:23.201106647Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:57196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:23.231341147Z","hostname":"sock","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"b5c8a2bb-cdc8-4db0-9eec-602cba301fab","remote_addr":"[fd00:1122:3344:101::1]:57196","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_11289ce8-4b64-40dd-90fe-89999cb8258a /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Address object not found","latency_us":30157,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:23.231603843Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:934","retry_after":"92.183215ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b5c8a2bb-cdc8-4db0-9eec-602cba301fab\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b5c8a2bb-cdc8-4db0-9eec-602cba301fab\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:23.355838761Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3437"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:30.947889419Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3455"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.643987677Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.644334684Z","hostname":"sock","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"4850f77a-2e1f-406f-941a-87be34cddcc8","remote_addr":"[fd00:1122:3344:101::1]:57196","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":12318538,"response_code":"204"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:35.653667885Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"984.752441ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.684329154Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.68511239Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38690"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.685850585Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.685985048Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.711759346Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.711809988Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.731219695Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.731270627Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.748704254Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.748752785Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.768556081Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.768600362Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.787596452Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.787637392Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.806649762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.806703414Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.826271505Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.826317766Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.844411657Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.844453948Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.92308715Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:35.923144272Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.002990209Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.0030472Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.078357135Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.078412526Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.149702388Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.149754229Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.223151954Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.223211835Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.295738422Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.295800784Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.370141088Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.370196129Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.442985302Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.443064654Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.50678218Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.506836612Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.5720853Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.572141991Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.638607164Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.638665695Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.702921303Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.703115677Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.703861082Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.704062626Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819098956Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819172217Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819180798Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819188038Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819194008Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819199808Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819205678Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819211458Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819218048Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819224198Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819229768Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819236768Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819242469Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819248119Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819253979Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819272319Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.81927804Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.81928346Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.8192903Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.819393722Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"00fa8d4c-77f6-4490-9ed0-4e48ac7cfbe1","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized)]","latency_us":1134184,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:36.819682288Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"138.060615ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"00fa8d4c-77f6-4490-9ed0-4e48ac7cfbe1\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"00fa8d4c-77f6-4490-9ed0-4e48ac7cfbe1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.958911313Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.95924387Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.95925127Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.981640539Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:36.98169131Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.003286293Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.003347924Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.024639271Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.024687462Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.045413097Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.045461298Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.065353896Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.065416168Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.084654102Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.084699273Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.105650233Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.105700044Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.124847466Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.124897357Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.145077161Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.145130222Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.163876786Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.163926057Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.184205713Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.184260204Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.202404636Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.202451777Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.221405506Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.221454267Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.241281623Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.241321534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.262255253Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.262311205Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.281717003Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.281772994Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.299737452Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.299778303Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.318765692Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.318806603Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.337378434Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.337418755Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.359246033Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.359301274Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.359492208Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.359678862Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.455065848Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.455127559Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.455136639Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.45514244Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.45514964Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.45515446Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.45516163Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.45516791Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.4551736Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.45517918Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.4551848Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.455190501Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.455197311Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.455201541Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.455208371Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.455232801Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.455237422Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.455243231Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.455247572Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.455327493Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"0553b3f6-561e-456a-b1c2-7b6481f45766","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized)]","latency_us":496276,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.455536208Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"80.711317ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0553b3f6-561e-456a-b1c2-7b6481f45766\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0553b3f6-561e-456a-b1c2-7b6481f45766\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.537733333Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.538033329Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.53804164Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.560204694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.560260145Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.580197824Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.580259835Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.599351597Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.599403768Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.619063651Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.619114202Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.636132471Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.636180632Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.654934367Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.654982028Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.672078609Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.672124219Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.689634169Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.689675539Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.707475435Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.707512745Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.725624147Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.725665837Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.744437702Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.744493174Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.761570894Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.761612145Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.780019822Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.780056343Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.797493081Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.797532571Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.81600691Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.816065371Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.835256915Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.835318197Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.853482329Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.85352808Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.871920127Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.871963548Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.890228963Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.890265453Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.907555148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.907604539Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.907824964Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.907964976Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998873611Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998920872Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998927882Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998935022Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998940702Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998946452Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998950662Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998957353Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998962973Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998967083Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998972303Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998978053Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998985193Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998990653Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.998996033Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.999010994Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.999016954Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.999023924Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.999029654Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:37.999141626Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"61b153e5-87b4-46de-9bae-cb508df94842","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized)]","latency_us":461295,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:37.999358721Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"137.845069ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"61b153e5-87b4-46de-9bae-cb508df94842\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"61b153e5-87b4-46de-9bae-cb508df94842\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.138567826Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.138887342Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.138894042Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.159647277Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.159700878Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.178677928Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.178724619Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.196536574Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.196582215Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.214769558Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.214809659Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.232780557Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.232833118Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.252933961Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.252978951Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.271308627Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.271350228Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.289845528Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.289888468Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.30948588Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.309531211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.33382981Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.333875531Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.355596566Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.355646277Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.375467744Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.375505894Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.393457152Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.393494923Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.412563584Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.412603915Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.432401831Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.432453092Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.45086088Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.450922921Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.469080274Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.469122604Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.486870918Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.486922669Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.504798226Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.504841337Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.522999199Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.52304608Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.523269595Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.523394388Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.612170278Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.612214479Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.612221129Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.612225779Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.612229989Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.612234169Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.612238339Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.61224277Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.61224705Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.61225127Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.61225533Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.61225956Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.61226396Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.61226732Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.61227164Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.61228982Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.612294491Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.612299521Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.612303711Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.612460804Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"04ac4109-8ec9-4299-9f14-7fc3ca924192","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized)]","latency_us":473751,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:38.612698819Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"111.004089ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"04ac4109-8ec9-4299-9f14-7fc3ca924192\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"04ac4109-8ec9-4299-9f14-7fc3ca924192\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.72489333Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.725278268Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.725283878Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.745775878Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.745802679Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.7634405Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.763480951Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.780426359Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.780455189Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.799062931Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.799090982Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.816335065Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.816367186Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.834532409Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.834563379Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.851896614Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.851929315Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.870336223Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.870364333Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.889160549Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.88920142Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.91164345Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.911675731Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.932192361Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.932244102Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.952882286Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.952911106Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.972341955Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.972371055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.990843394Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:38.990875085Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.010802053Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.010829324Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.032032299Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.03207122Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.05355537Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.053588491Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.071173022Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.071209023Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.092568341Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.092597921Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.112249244Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.112286005Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.112494759Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.112666313Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.211979669Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.21204774Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.21205532Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.21206523Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212074531Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212082871Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212090881Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212099361Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212103271Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212108711Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212114191Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212119681Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212126881Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212132212Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212137792Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212152532Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212158472Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212163822Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.212169372Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.212306365Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"1814de4a-2776-409e-86c2-1ee62b653abe","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized)]","latency_us":487249,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.21254364Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"138.295276ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1814de4a-2776-409e-86c2-1ee62b653abe\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1814de4a-2776-409e-86c2-1ee62b653abe\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.352703104Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.353018321Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.353029031Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.375543703Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.375584024Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.394385159Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.39442115Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.41292647Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.4129665Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.430754385Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.430791826Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.448466359Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.44851134Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.466142071Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.466176472Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.485301254Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.485338365Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.50312514Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.50315964Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.521521327Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.521554298Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.539872043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.539908244Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.557964164Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.558010895Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.576116157Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.576153617Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.593923332Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.593957242Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.611913191Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.611950892Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.630759747Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.630807898Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.651187346Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.651245157Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.670816729Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.67086211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.68940577Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.689447681Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.708755397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.708802598Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.726380538Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.726426809Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.726631333Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.726756106Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.81616824Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.81620337Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.81620987Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816214351Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816218821Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816223191Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816227381Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816231771Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816236641Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816240941Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816245231Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816249391Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816253602Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816257002Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816261842Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816279492Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816283852Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816287842Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.816291962Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.816455075Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"91ceb597-691c-4d2c-ab65-b7d088dbb9b8","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized)]","latency_us":463589,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:39.81665509Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"158.034736ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"91ceb597-691c-4d2c-ab65-b7d088dbb9b8\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"91ceb597-691c-4d2c-ab65-b7d088dbb9b8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.975811594Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.97610583Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.97611187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.995419386Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:39.995456707Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.013750332Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.013815123Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.032376484Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.032414705Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.050255501Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.050287841Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.067910863Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.067947233Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.085674697Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.085715608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.105212978Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.105249218Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.122927601Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.122964592Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.142039603Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.142083934Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.160375018Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.160411779Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.179029131Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.179074792Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.197914848Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.197946929Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.215739974Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.215773884Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.234640071Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.234675772Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.253690872Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.253731463Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.273312124Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.273359245Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.294863336Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.294904727Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.316534931Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.316579272Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.338094693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.338137904Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.358268487Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.358316998Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.358509852Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.358644954Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.466092848Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.466160199Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.46616747Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.46617358Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.46617953Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.46618545Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.46619102Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.46619665Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.46620208Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.46620757Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.466211711Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.466218711Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.466224411Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.466230411Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.466235821Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.466253431Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.466260972Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.466266702Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.466272752Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.466436855Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"e8cf5fe5-b827-45c0-8b89-54d23c39627c","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized)]","latency_us":490479,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:40.466733741Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"316.840887ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e8cf5fe5-b827-45c0-8b89-54d23c39627c\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e8cf5fe5-b827-45c0-8b89-54d23c39627c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.784126871Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.784435677Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.784444147Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.806481029Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.8065318Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.827255215Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.827304216Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.848530851Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.848581162Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.868753746Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.868800607Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.88944036Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.889506282Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.911109865Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.911172446Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.932404601Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.932457903Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.954091636Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.954144457Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.97526226Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.975312132Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.996887034Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:40.996936045Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.020126371Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.020185162Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.040948058Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.041001319Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.064860858Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.064909159Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.087436721Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.087486022Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.108886131Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.109081825Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.130704148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.130757889Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.152804501Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.152855762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.174857403Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.174923975Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.197309464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.197360245Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.219778565Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.219845666Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.220009369Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.220135532Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320814847Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320881568Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320888608Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320894758Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320899128Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320904339Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320909629Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320914949Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320921819Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320927489Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320932979Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320938539Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.32094582Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320951549Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.32095704Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.32098126Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.32098835Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.320993761Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.32099779Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.321091623Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b6ba03f6-fd63-42e2-9fbc-6695e6da468c","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized)]","latency_us":536833,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:41.321327407Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"411.370445ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b6ba03f6-fd63-42e2-9fbc-6695e6da468c\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b6ba03f6-fd63-42e2-9fbc-6695e6da468c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.733928589Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.734252776Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.734260056Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.757715207Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.757765018Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.77981142Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.779864571Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.800644187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.800691399Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.819633497Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.819675448Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.840975465Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.841021875Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.862632929Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.86267472Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.882582748Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.882620619Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.901824982Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.901867963Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.921604258Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.921644179Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.94316373Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.943209331Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.965511199Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.96556756Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.986959299Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:41.987001789Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.009801637Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.009846548Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.029693655Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.029732876Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.050670645Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.050713586Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.071673696Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.071717497Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.093128736Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.093167687Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.113165867Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.113213718Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.133567765Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.133607846Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.15480217Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.154855611Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.155044805Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.155164937Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.25280018Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252850681Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252856751Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252860231Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252864471Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252869611Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252873001Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252877061Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252881252Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252885422Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252889912Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252894792Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252898952Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252904062Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252916132Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252921302Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252925582Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252930393Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.252933583Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.253026924Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"a6b186ff-6254-4e25-a8f8-81e32eaed181","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized)]","latency_us":518958,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.253250509Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"228.135157ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a6b186ff-6254-4e25-a8f8-81e32eaed181\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a6b186ff-6254-4e25-a8f8-81e32eaed181\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.482708185Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.483040172Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.483048872Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.506913302Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.506952922Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.527211588Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.527246659Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.545612475Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.545650246Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.565502753Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.565536604Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.585160756Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.585201847Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.605982403Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.606015434Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.626413222Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.626445373Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.645865621Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.645900602Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.665037354Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.665068695Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.685901872Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.685932653Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.706333171Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.706379662Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.726014785Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.726048786Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.746299061Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.746332332Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.765087026Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.765127937Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.786348902Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.786384753Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.806325112Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.806356783Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.827505116Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.827540417Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.849797654Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.849833994Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.87009649Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.870129951Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.892335056Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.892382957Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.892594071Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.892732244Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990746204Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990801805Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990808006Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990813636Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990818706Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990823926Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990829156Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990834296Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990837896Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990843096Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990848616Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990853707Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990858907Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990864007Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990884517Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990888447Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990892127Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990897147Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.990900657Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:42.99100673Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"0caf80db-5978-4ed3-82d2-c18286bafa8c","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized)]","latency_us":508131,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:42.991296285Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"295.2224ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0caf80db-5978-4ed3-82d2-c18286bafa8c\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0caf80db-5978-4ed3-82d2-c18286bafa8c\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.201758221Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60664"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.202386814Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"8bf8f3c1-0a60-4ab3-b0b4-2a39ee929737","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60664","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":55,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.287708024Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.28801891Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.28802615Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.311361579Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.3114202Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.335138206Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.335189097Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.355922393Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.355971574Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.37480234Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.374847031Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.396503505Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.396544296Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.417449564Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.417490775Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.43770421Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.437749531Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.457656769Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.4577035Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.477957775Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.478001396Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.500764103Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.500818884Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.52354232Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.523582431Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.54450699Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.544546291Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.565065912Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.565109953Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.585895339Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.58593531Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.607100564Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.607141145Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.628466362Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.628506043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.647934451Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.647978172Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.668018723Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.668061214Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.688043454Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.688087145Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.709121566Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.709168217Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.709387642Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.709527445Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809424813Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809476135Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809481664Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809485095Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809490195Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809494305Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809498355Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809503185Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809507405Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809511355Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809516445Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809529276Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809534566Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809538976Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809543306Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809546526Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809549676Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809553766Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809556896Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:43.809634338Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"783b165c-6990-4f5d-a176-e1e3f63df099","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized)]","latency_us":521793,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:43.809839552Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"520.51667ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"783b165c-6990-4f5d-a176-e1e3f63df099\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"783b165c-6990-4f5d-a176-e1e3f63df099\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.331546781Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.331877148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.331884218Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.355195336Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.355236907Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.377879181Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.377916802Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.397626096Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.397667217Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.417516284Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.417551095Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.439610997Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.439648398Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.458996445Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.459032005Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.477054335Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.477086886Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.497062785Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.497097406Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.516134947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.516168957Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.534375081Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.534424192Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.553624736Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.553659786Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.5723802Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.572422841Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.592440261Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.592476272Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.611590064Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.611623655Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.630594594Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.630629615Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.651804269Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.65184018Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.671461902Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.671494863Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.691461632Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.691496363Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.711091625Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.711128636Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.731553774Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.731595315Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.731788949Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.731911042Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.829267309Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.82932111Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.82932935Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.82933493Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.82934019Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.82934576Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.82935116Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.82935667Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.82936156Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.829366771Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.829372291Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.829398041Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.829402401Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.829406291Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.829411581Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.829415252Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.829418862Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.829422672Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.829426302Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:44.829507063Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"e111aef6-ef2a-446d-a06f-2005c2bc8343","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized)]","latency_us":497805,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:44.829734418Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"880.366376ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e111aef6-ef2a-446d-a06f-2005c2bc8343\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e111aef6-ef2a-446d-a06f-2005c2bc8343\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.710858678Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.711152924Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.711160144Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.735065904Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.735116646Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.75679072Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.756844101Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.77628589Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.776334491Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.795761939Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.7958094Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.81530424Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.815353751Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.837817842Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.837869993Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.860363014Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.860410295Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.878829203Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.878872094Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.899101278Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.89914669Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.918310412Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.918367364Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.93867052Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.938711761Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.958308323Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.958354524Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.978051238Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.978101899Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.998393225Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:45.998440806Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.020179492Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.020219733Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.040828185Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.040867646Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.062184273Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.062226834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.081850777Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.081889047Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.102679834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.102718245Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.123288306Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.123332627Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.123533801Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.123647314Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224591213Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224663065Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224671435Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224677605Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224683265Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224690415Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224694705Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224701555Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224707106Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224712426Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224717866Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224730846Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224736616Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224740606Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224744496Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224748506Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224754016Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224757636Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.224762667Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.224830388Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"63520786-f575-4d27-80dc-d55f98c5bd77","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized)]","latency_us":513843,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:46.225048893Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"654.389284ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"63520786-f575-4d27-80dc-d55f98c5bd77\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"63520786-f575-4d27-80dc-d55f98c5bd77\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.880398973Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.881284581Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.881295521Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.902934375Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.902988806Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.922523927Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.922573918Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.941999576Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.942057307Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.961755851Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.961801663Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.980287122Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.980346663Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.999278101Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:46.999325552Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.02066433Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.020725611Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.039753781Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.039805462Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.058926164Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.058973205Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.07775909Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.077811992Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.098843993Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.098890614Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.11772734Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.117776651Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.136617808Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.136669519Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.157183779Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.15723452Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.177752571Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.177806662Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.1981984Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.198254221Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.217540847Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.217584978Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.237671149Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.237722531Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.257925935Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.257979996Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.279864055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.279932366Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.2800998Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.280212432Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376383534Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376429605Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376435866Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376440126Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376444536Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376449516Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376453756Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376457796Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376461836Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376465946Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376470166Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376492127Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376495767Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376498977Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376502487Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376505677Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376508787Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376512857Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.376515987Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:47.376573268Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"00023f6f-87fd-4bb1-bb53-a92e8d95655a","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized)]","latency_us":496040,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:47.377300083Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"818.66203ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"00023f6f-87fd-4bb1-bb53-a92e8d95655a\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"00023f6f-87fd-4bb1-bb53-a92e8d95655a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.19782457Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.198350441Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.198363942Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.220596397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.220633988Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.240627738Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.240663599Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.26021179Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.260261621Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.279398773Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.279429294Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.296964754Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.297015915Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.316899353Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.316931493Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.335023314Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.335051465Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.353281289Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.353312979Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.371691866Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.371717387Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.390544803Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.390587694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.408105903Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.408133433Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.426193594Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.426224804Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.444499199Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.44452719Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.46305459Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.463088001Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.482238173Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.482268074Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.500737013Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.500777544Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.518722992Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.518747232Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.536872854Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.536899424Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.555172249Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.55521205Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.574005155Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.574067827Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.574317352Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.574479875Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668571845Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668632126Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668640266Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668647386Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668656896Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668665927Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668673167Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668682117Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668691457Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668698977Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668723178Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668731068Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668736418Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668743838Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668749078Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668754338Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668760169Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668767329Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.668776269Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:48.66883632Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"5a84ba31-46d4-424a-8f9f-76b65fa62423","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized)]","latency_us":470806,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:48.669075335Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.2271172s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5a84ba31-46d4-424a-8f9f-76b65fa62423\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5a84ba31-46d4-424a-8f9f-76b65fa62423\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.897769303Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.898066229Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.89807402Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.919644672Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.919683743Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.938382226Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.938422047Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.954516407Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.954559308Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.972702Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.972735851Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.988596576Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:49.988641317Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.006249088Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.006290009Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.022984211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.023023982Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.039199154Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.039244035Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.058278825Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.058320326Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.07652334Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.07657795Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.095325185Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.095362506Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.115078Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.115114341Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.13359146Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.133638571Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.151271042Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.151312483Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.169155448Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.169194959Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.186515244Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.186556365Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.203960322Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.203996223Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.220818138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.220851059Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.238614003Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.238652124Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.256301675Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.256343166Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.256556731Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.256683633Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.344713469Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.34475234Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.344756529Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.34476075Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.34476398Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.34476803Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.34477216Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.34477718Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.34478126Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.34478531Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.34480371Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.344808041Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.344812231Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.344816471Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.344820601Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.344825551Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.344829871Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.344834011Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.344838051Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:50.344920023Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"c2e31bc8-dbf2-4c58-9052-6f8144b538c0","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized)]","latency_us":447029,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:50.345114617Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.097299393s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c2e31bc8-dbf2-4c58-9052-6f8144b538c0\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c2e31bc8-dbf2-4c58-9052-6f8144b538c0\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:51.686961056Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.818697767s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.443356688Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.443754186Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.443766206Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.464633304Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.464674855Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.48442416Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.484459651Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.50390577Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.50394213Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.523718736Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.523754117Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.542823968Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.542858939Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.560108752Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.560138763Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.577878067Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.577915187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.595367685Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.595401926Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.613298013Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.613343704Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.631877294Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.631908855Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.650018906Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.650053767Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.667893243Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.667927724Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.685202228Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.685233428Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.703407921Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.703447742Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.721743607Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.721778808Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.739208815Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.739239396Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.75650111Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.756536821Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.774930468Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.774962419Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.792772524Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.792805195Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.810190441Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.810247492Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.810519858Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.810702072Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902410542Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902465984Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902474744Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902483254Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902489454Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902497494Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902503624Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902509465Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902525955Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902532325Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902540235Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902546235Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902552385Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902560016Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902565636Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902571836Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902576476Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902582336Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902586796Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:52.902684928Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"5446ce6d-1846-46fd-bdb6-20223dab4abd","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized)]","latency_us":459140,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:52.902935573Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.252327609s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5446ce6d-1846-46fd-bdb6-20223dab4abd\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5446ce6d-1846-46fd-bdb6-20223dab4abd\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.156297527Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.157273497Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.157281727Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.178730227Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.178769548Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.196084413Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.196119643Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.214890608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.214934939Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.233872258Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.233908839Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.253055561Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.254981831Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.270831576Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.270878367Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.288163141Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.288207372Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.304131268Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.30418519Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.321406353Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.321457654Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.337985643Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.338024144Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.355980792Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.356033483Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.372538022Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.372580312Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.389451598Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.38950559Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.406855355Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.406918537Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.423892455Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.423942836Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.440320322Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.440360982Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.456385401Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.456432492Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.471751716Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.471789737Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.489615023Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.489654933Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.506265284Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.506324555Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.506498649Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.506614341Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.594893342Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.594946933Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.594954363Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.594965823Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.594971463Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.594978053Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.594989414Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.594996134Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.595019524Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.595026694Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.595037625Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.595044605Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.595051055Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.595057865Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.595063085Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.595067635Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.595074026Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.595079515Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.595084896Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:54.595183258Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"10962b36-18a5-41b0-a5ab-9c2b24fa7673","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized)]","latency_us":438682,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:54.595357651Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.427234877s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"10962b36-18a5-41b0-a5ab-9c2b24fa7673\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"10962b36-18a5-41b0-a5ab-9c2b24fa7673\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.02362256Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.024122681Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.024132681Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.045606471Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.045659803Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.064645262Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.064694963Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.082270763Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.082314404Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.100436846Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.100482667Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.117597728Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.117642609Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.134173258Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.134219089Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.150089414Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.150130155Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.168137134Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.168177164Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.187063172Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.187124073Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.206327607Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.206368148Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.22454571Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.224588981Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.242286214Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.242337445Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.259343174Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.259404885Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.278842154Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.278882115Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.297955076Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.297996567Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.31764561Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.317688561Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.334629468Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.334672459Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.353797241Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.353843952Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.371729819Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.37177349Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.390357441Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.390398292Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.390651867Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.39081408Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.481945049Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482018321Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482027901Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482036391Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482045492Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482053781Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482060152Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482069122Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482088782Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482103783Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482112153Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482118693Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482133953Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482139263Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482150823Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482156154Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482160954Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482172764Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.482179664Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:57.482244805Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"cab4c901-f1e8-4b66-a7b5-e2a5c65e29dc","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized)]","latency_us":458320,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:09:57.48246694Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.347661479s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cab4c901-f1e8-4b66-a7b5-e2a5c65e29dc\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cab4c901-f1e8-4b66-a7b5-e2a5c65e29dc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.83129192Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.831667338Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.831683848Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.85273194Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.855123379Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.8707912Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.870853041Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.888971793Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.890902333Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.90732641Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.907385101Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.925248577Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.925300978Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.942727456Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.942780857Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.959806306Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.959870477Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.976628651Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.976690262Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.995756453Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:09:59.995815874Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.016717153Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.016775924Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.034414126Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.034466967Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.052060888Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.054307944Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.071026497Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.071091458Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.087984864Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.088039146Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.105756619Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.10581107Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.123947112Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.124002923Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.141242457Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.141297698Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.159275936Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.159332197Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.176150072Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.176211743Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.193854535Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.193927886Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.194184281Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.194323864Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282037723Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282094655Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282105295Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282117085Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282123975Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282134995Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282141725Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282147396Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282168866Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282176256Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282186736Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282193426Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282204557Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282211137Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282216737Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282223217Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282228697Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282235237Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282241647Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:00.282333779Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"368bd38e-68d8-4549-9173-dac6f4173cf0","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\", TimeNotSynchronized), (\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\", TimeNotSynchronized), (\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\", TimeNotSynchronized), (\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\", TimeNotSynchronized), (\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\", TimeNotSynchronized), (\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\", TimeNotSynchronized), (\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\", TimeNotSynchronized), (\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\", TimeNotSynchronized), (\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\", TimeNotSynchronized), (\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\", TimeNotSynchronized), (\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\", TimeNotSynchronized), (\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\", TimeNotSynchronized), (\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\", TimeNotSynchronized), (\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\", TimeNotSynchronized), (\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\", TimeNotSynchronized), (\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\", TimeNotSynchronized), (\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\", TimeNotSynchronized), (\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\", TimeNotSynchronized), (\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\", TimeNotSynchronized)]","latency_us":450879,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:00.282541734Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"4.317831529s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"368bd38e-68d8-4549-9173-dac6f4173cf0\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:09:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"368bd38e-68d8-4549-9173-dac6f4173cf0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 08a6de18-4447-48d4-a83c-cd709a350ec5, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: d21c386f-fd0d-4f7c-a479-36f35379ea0c, kind: Service { id: b5d37b99-7ecd-4934-8190-c9ab1cf73eb6 }, name: Name(\"ntp-b5d37b99-7ecd-4934-8190-c9ab1cf73eb6\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 160, 110])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 11289ce8-4b64-40dd-90fe-89999cb8258a, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: c369e252-d803-47dd-b8d9-99560ba33cb1, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6398723a-debc-46be-9f6c-b8deb81778ac, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: a21507ee-5c96-4540-bc06-0852e4850905, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: be1fa9a1-301b-46b9-b30a-f282967efa55, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 51231501-67c9-451e-a935-83aa38e09044, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: bab6e283-4f89-4430-8daa-991557899d2b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: cf421c38-9400-403d-b409-822a08e6d3df, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d3a9225c-107a-4f4b-a8d2-1285aaef40cb, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External } } } }, OmicronZoneConfig { id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External } } } }, OmicronZoneConfig { id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External } } } }, OmicronZoneConfig { id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External } } } }, OmicronZoneConfig { id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External } } } }, OmicronZoneConfig { id: 398377fe-8f6c-470f-9364-6ed47bb8001f, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External } } } }, OmicronZoneConfig { id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External } } } }, OmicronZoneConfig { id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.601855854Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.602250453Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.602265333Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08a6de18-4447-48d4-a83c-cd709a350ec5, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.623000808Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.623039919Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b8b4505b-bc72-4995-b8bc-e81e97f16f1f, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.643531349Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.64359369Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d8f0884b-aee3-45e7-9763-c04c8a43e0ba, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.663609411Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.663645792Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 11289ce8-4b64-40dd-90fe-89999cb8258a, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.683641062Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.683695033Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b4fbfbe3-741c-42e7-9901-7e3852a09f98, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.703186703Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.703227764Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 119d06a5-5604-4ebd-b3fb-bf4627742d2d, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.722036279Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.7220806Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c369e252-d803-47dd-b8d9-99560ba33cb1, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.74060303Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.740651981Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6398723a-debc-46be-9f6c-b8deb81778ac, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.759049298Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.759096219Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b07b723-29d0-44f0-aa06-d84f7872ea97, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.777421775Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.777468336Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc451e9c-ecba-4d54-8c4b-8f9b74446294, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.795784882Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.795820332Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 39d44799-bf67-4a04-a01e-2dab5d43df6e, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.813946594Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.814006896Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511, dataset_name: DatasetName { pool_name: ZpoolName { id: d1b1e109-515b-482d-930d-ba493cd78ef0 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.832194209Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.832232829Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 420b93bc-5ca2-4380-8284-8eebea6a5aa1, dataset_name: DatasetName { pool_name: ZpoolName { id: 7b71d6bd-a486-4799-a3b4-23ca97ac0816 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.850440783Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.850485104Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d17ede40-1516-4fc0-9d4a-6777ee2cc962, dataset_name: DatasetName { pool_name: ZpoolName { id: a1b82a77-810b-4e64-9e93-3bddcb08ae61 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.869020264Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.869071505Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7, dataset_name: DatasetName { pool_name: ZpoolName { id: 24a77df0-4435-4add-9951-0211a2cf4c49 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.887122255Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.887158556Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d5f9bf80-883f-4acc-a35d-a7494c2830d6, dataset_name: DatasetName { pool_name: ZpoolName { id: 6686ffc9-9b48-48c1-988b-7aac0a49e5a5 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.905186475Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.905221526Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 398377fe-8f6c-470f-9364-6ed47bb8001f, dataset_name: DatasetName { pool_name: ZpoolName { id: 92f3b999-4455-49cb-8ded-aa658c2709d6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.922618323Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.922667844Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b1369ec1-e8a8-48c1-9a6c-96e11e05bc20, dataset_name: DatasetName { pool_name: ZpoolName { id: feef97bb-ca23-472f-b173-ce352fd7d46c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.941215154Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.941267106Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffa1733-2919-4d64-8165-6d72c0607851 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.958834296Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.958877647Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2d06eca8-fecb-46e7-b153-2aab6db47cf8, dataset_name: DatasetName { pool_name: ZpoolName { id: 05c910b1-d5d8-4b62-8b1a-0afed5d62eeb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.977610691Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.977666112Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.977948088Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:04.978120471Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.072330823Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.072740592Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.111244081Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.111480136Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.111522657Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.111560928Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.115729973Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.115845246Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.115878476Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.13946377Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.139695515Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.284441443Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.355258385Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.355314706Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.355320376Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.355328196Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.355331766Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.355335897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.355339446Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.355439628Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.360335529Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.36527626Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.370348894Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.376229075Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.381278338Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.386080297Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.524777041Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.52957614Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.533776676Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.537907511Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.542811651Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.54747147Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.553017Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.558682567Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.563497946Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.568066079Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.572603673Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.577805619Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.583122628Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.588380806Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.594860309Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.600558466Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.605078538Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.609626432Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.614089793Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.618545894Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.622978215Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.666818414Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.700927444Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.734918771Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.768820516Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.80214942Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.803703601Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.803711502Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.857106827Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 150, 20]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.857169268Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.859411354Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.859445155Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.912692087Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 154, 143]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.912773708Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.914729278Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.914756009Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.977851183Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 145, 254]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.977908504Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.979850324Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:05.979878145Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.03524493Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 131, 109]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.035310081Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.037258851Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.037285442Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.091072085Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 163, 90]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.091132626Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.095018276Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.095046947Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.095333512Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.095350993Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.095476645Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.095485356Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.095559627Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.095566517Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.095765531Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.095772891Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.098004507Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.100822265Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.100848625Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.102960719Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.1054627Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.10548347Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.105493211Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.107797848Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.110031044Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.110043934Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.232512345Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.237976367Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.24250873Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.247350149Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.252681348Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.25908021Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.266116308Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.272051466Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.278642311Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:06.284793647Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:08.547175562Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"5.592284437s"} {"msg":"Profile for oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.312565888Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.312799993Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.313344684Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.320500281Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.326902532Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.327072846Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.553662533Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.553843316Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.553910367Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.561941342Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.56285947Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.563032049Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.563047724Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.570187841Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.570329944Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.570646941Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.570877375Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.571021718Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.57693216Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.582909592Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.588756412Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.594668403Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.599948442Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.605586207Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.611662692Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:09.611901937Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 33.331413ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.410992903Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 69.769867ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.419877066Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 29.636395ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.427986832Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 34.678401ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.437309443Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 49.271779ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.446538523Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 37.377884ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.454907753Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 56.853138ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.462960829Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 25.08189ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.471126147Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 71.795765ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.481281815Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 35.546254ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.60732665Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 32.599666ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.613627629Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 62.726751ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.645274878Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 69.768569ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.733862025Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 135.976617ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.76091593Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 131.276793ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.880024793Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 109.021725ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.900176596Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 142.613323ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.921274468Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 126.660577ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.938161645Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 52.438779ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.951515419Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 62.832753ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.968254362Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 94.942819ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:10.984662328Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 92.385816ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.003468624Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 109.31943ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.026439965Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 80.068651ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.046110679Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 74.944355ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.062053175Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 82.228874ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.080140606Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 275.820678ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.098222697Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 166.114478ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.114564712Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 219.747392ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.130677523Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 106.006661ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.152312246Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 104.101717ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.16813193Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 160.807805ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.177977022Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 125.187523ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.187210341Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 220.253188ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.198046714Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 161.014701ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.207493628Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 233.328163ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.217082924Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 271.309316ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.229613171Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 287.179797ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.240023445Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 128.390442ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.251449079Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 225.399761ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.268102631Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 242.974604ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.283322843Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 348.963655ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.294650565Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 596.251166ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.324855044Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 240.63338ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.353995182Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 563.418957ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.365861425Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 318.445335ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.382771062Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 471.774929ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.394253088Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 331.088579ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.406879686Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 245.44968ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.429551371Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 268.721773ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.463589009Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 543.1482ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.506140902Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 404.293082ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.519953916Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 1.030830394s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.537024646Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 273.040351ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.547114093Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.094347745s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.606612413Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 551.47334ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.658466986Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 479.09987ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.690157086Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 489.732848ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.715956165Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 742.059639ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.7468999Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 619.420558ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.758759203Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 445.625512ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.83369362Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 480.433501ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.879350896Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 645.350805ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.936470997Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 411.810736ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.950047766Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 905.810495ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:11.967501814Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 544.046512ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.064705737Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 1.16165271s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.185191057Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.034853433s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.219079552Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 610.429518ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.23163771Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 1.006779427s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.294342926Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 621.429345ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.373002759Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 1.452580734s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.389313643Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 1.388566348s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.404963284Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.330456827s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.507071188Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 1.335548348s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.58030918Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.163552458s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.593376278Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.020966482s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.621363412Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.036081619s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.713700176Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 691.430038ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.856689328Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.448662615s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:12.88798159Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 924.477539ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.007608852Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.339665086s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.267216407Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 1.309442488s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.316066818Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:13.316454436Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64908"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:13.316668681Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"28896144-3739-4f51-a162-c497ed2d716a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64908","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":123,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 524.116587ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.361556211Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 1.399615929s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.563533493Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.042972257s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.656653703Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.348245454s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.76327662Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 549.439072ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.774411728Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 1.483145664s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.808008697Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 554.371577ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.851185087Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 943.385072ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.863908804Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 737.869859ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.89739978Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 516.627586ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.924410731Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 1.216657402s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:13.941471514Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 980.982279ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:14.516317823Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.323584382s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:14.532229329Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.180508328s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:14.572154997Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.419616022s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:14.603844998Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.432147764s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:14.6395829Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 1.475387146s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:14.664822838Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 852.25004ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:14.676102209Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 650.59528ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:14.71415729Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 1.187129053s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:14.822841559Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 653.224889ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:14.974304585Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.308316261s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:15.122156837Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 1.261612847s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:15.168335203Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 886.6241ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:15.301915413Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 776.252289ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:15.374210825Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 604.243218ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:15.507331585Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 616.225982ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:15.538891212Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 1.025541024s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:15.636179109Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.023127789s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:15.761436986Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.025192296s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:15.86452738Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 698.601554ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.019655542Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 706.845214ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.033081097Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.125175802s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.07955665Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 893.508611ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.120748835Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 942.775871ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.150022975Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 637.824828ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.158379616Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 1.286063997s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.165588364Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 811.767403ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.199371537Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 903.845613ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.491531889Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.361271502s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.548529457Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 1.181656276s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.70326Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 1.169341555s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.768174262Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 933.600053ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.779507154Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.495711778s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.79683643Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.197779001s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.808856056Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 963.542872ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:16.901167989Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 807.707582ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.023169951Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 876.805049ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.033172446Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 525.706295ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.102799774Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.470485423s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.215625167Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 595.948785ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.409146716Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 630.457863ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.46348228Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 861.676506ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.64240916Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 552.720917ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.727067196Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 697.958792ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.844095876Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.097069687s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.878020002Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 663.577181ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.896289926Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 894.597391ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.923146197Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.491191593s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.934540341Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 1.05345218s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:17.9501185Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 730.019412ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:18.018013202Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 705.360217ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:18.030820155Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 1.303022274s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:18.109417097Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.100702099s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:18.293990312Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 959.090123ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:18.305743613Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 1.335456759s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:18.518557867Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 1.09503425s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:18.557114057Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 1.478690651s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:18.57186747Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.43186335s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:18.700215632Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 613.173607ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:18.748687216Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 504.327206ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:18.760009529Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 1.377069827s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:18.927010823Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.302437808s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:19.040543222Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 1.21627295s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:19.054039559Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 721.923211ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:19.27651048Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.275809235s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:19.287479045Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.007001505s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:19.372996939Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 811.08444ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:19.408973037Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 637.342332ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:19.426854093Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 752.217129ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:19.439422031Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 1.16365275s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:19.664037698Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 1.301847809s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:19.86600328Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 1.321711951s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:20.010134455Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 1.402498315s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:20.06206112Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 1.074217743s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:20.07376032Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.229877091s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:20.142107052Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.360746327s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:20.201093961Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 930.880503ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:20.227492532Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 872.361545ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:20.279754954Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 1.468642933s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:20.30295717Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.003100445s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:20.350694339Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.359783765s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:20.388375551Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.164651375s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:20.573013718Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 661.588597ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:20.856665125Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:20.884569197Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_51231501-67c9-451e-a935-83aa38e09044:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:20.885388114Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:20.885431725Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:20.889964492Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 49.357895ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.056297669Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 63.038184ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.063688118Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 60.715104ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.114186496Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 125.040744ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.134687967Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 744.93834ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.157153097Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 1.174611064s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.164397176Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.220181456s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.170839617Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 1.35689371s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.177035865Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 261.215192ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.183528308Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 151.894324ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.270799698Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 824.650971ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.341029498Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.271426279s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.3635493Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.456256794s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.380804914Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 594.883505ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.431745768Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 528.094219ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.456894724Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 551.502266ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.473961484Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 1.274201834s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.529014163Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.307519571s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.571056625Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.163569346s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.747661137Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 640.375755ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.75660115Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 1.280722422s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.780790566Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 1.29778059s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.910986096Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 749.29945ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:21.996040561Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 1.193535675s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:22.03501209Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 552.141308ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:22.043215778Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 588.553877ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:22.173844286Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 957.992112ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:22.348878806Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.153194047s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:22.41152236Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.233629248s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:22.420186058Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 885.537542ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:22.617506935Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 1.407086935s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:22.631732976Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 880.291731ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:22.660136679Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 786.424877ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:22.869780278Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 690.188693ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:22.931513895Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.216857212s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:22.954835013Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 886.579724ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.032836142Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 666.901156ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.09514848Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.25671472s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.106408381Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 975.361123ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.116167631Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 1.240938798s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.219885711Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 1.140349521s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.238147232Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 899.634919ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.316734184Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 805.840423ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.513000279Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.061940944s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.550292524Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 544.740963ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.57452483Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 1.420057592s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.630818665Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 541.138886ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.663955164Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 1.168834372s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.672303316Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.147542542s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.771650103Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 1.024578574s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:23.925681792Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 1.325792419s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.049832528Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 858.476825ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.101818804Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 747.78507ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.128031722Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:24.143282945Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:24.143396867Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.374946388s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.182970328Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.420007578s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.214019935Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 941.791138ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.225391148Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 738.029616ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.328424711Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.300359274s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.371597466Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 715.530427ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.388817919Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 63.338353ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.450567986Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 800.677173ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.469521224Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 80.82088ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.521733925Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 223.281441ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.61461773Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.243190339s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.622974061Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 233.345927ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.851727973Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 935.270851ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.862777949Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.444440972s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.888063488Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:24.900759738Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:24.900953002Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 915.664287ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.930900553Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 1.313109449s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.961664047Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 878.377724ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:24.972272625Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:25.010942038Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:25.011022599Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 529.335252ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.06274997Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 946.319292ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.076135455Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 902.842158ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.095022812Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 1.417160498s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.114250146Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 70.372682ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.121676049Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 923.159426ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.176128895Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 53.082047ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.183133648Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 91.225885ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.200101706Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 96.308447ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.244487627Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 1.488543327s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.278745171Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 182.152388ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.302735936Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 144.259359ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.350863358Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 975.614431ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.387342936Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 471.176074ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.495044195Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 594.274621ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.505739214Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 564.697058ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.568127304Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 725.173329ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.60255196Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.111534794s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.643478219Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.495018419s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.681303935Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 595.477587ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.823899329Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 687.321851ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.891175679Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.169749393s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.909510295Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 692.159671ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.929380692Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 1.02757349s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:25.981205375Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 1.151198288s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.031918385Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 774.593697ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.050802772Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 721.916129ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.116921599Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 801.56491ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.125643827Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.048858633s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.143207547Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 614.235591ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.287112948Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 826.564636ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.337466981Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.114190969s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.34525339Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 631.004884ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.371904347Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 868.638765ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.425529516Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 728.575502ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.54029843Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 942.9225ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.588621981Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 858.914622ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.630118492Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.059176033s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.764850525Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 1.300838953s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.775098145Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 978.356675ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.833791649Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 537.726643ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.847088622Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 1.182678712s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.910254527Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 1.357439472s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:26.936417814Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 527.940034ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.013177307Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 526.365488ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.020991428Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.046359762s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.086694326Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 1.387646119s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.172569566Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.164830162s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.184188004Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 1.387154504s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.191650437Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.486602967s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.199650302Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 1.350251393s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.277389336Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 538.731845ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.301732865Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 1.338455657s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.393487457Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.011198799s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.46777475Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 517.472735ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.497490639Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 966.69929ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.538513691Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 1.244358129s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.550805823Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 1.403036903s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.558356248Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 1.156334041s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.820513114Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 585.116503ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.832771415Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 1.26891853s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:27.848554299Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.442494227s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.03438278Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 1.33712269s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.085617731Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 709.220885ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.101414135Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.086898623s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.143201182Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 1.1856587s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.307386758Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.385502262s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.363195692Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.250516138s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.438562608Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.425762298s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.560751424Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 1.089430275s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.57568971Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 1.069605533s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.585769467Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 1.186220881s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.594295522Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:28.626735467Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:28.626928241Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 1.394882838s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.651930784Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.231389712s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.706061204Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 1.000264257s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.753569758Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 1.081474021s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.814034438Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 916.954754ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.834549889Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 1.346737952s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.982048664Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\") failed: Property not found. retry in 47.720834ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:28.99894831Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 1.241093024s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.024649557Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\") failed: Property not found. retry in 73.483114ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.075680334Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 962.132931ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.230048179Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\") failed: Property not found. retry in 101.133618ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.291608282Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.184261385s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.320590076Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\") failed: Property not found. retry in 572.139742ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.410383267Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 606.238717ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.470128333Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 921.313268ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.504357105Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.518204919Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"8.482752464s"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 502.818646ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.523338214Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 1.02834457s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.676651568Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 751.230919ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.693926772Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 686.863065ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.71674909Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.384883855s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.763814416Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 890.276728ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.775538986Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 807.98653ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.78596998Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 907.002309ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.797727631Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 620.366481ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.908334219Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 998.525576ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.949991614Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\") failed: Property not found. retry in 603.92925ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:29.994753172Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 582.652276ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.005045463Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 1.423261611s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.040266215Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 1.145399631s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.060066051Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 931.250761ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.097774044Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 632.98176ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.206736148Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 728.415864ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.277073181Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 1.208630645s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.345273219Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.237444462s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.420247497Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.176030026s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.440936891Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 913.564984ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.458645309Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.483677924s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.518833809Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 1.052540632s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.540656616Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.075909805s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.599190527Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\") failed: Property not found. retry in 1.292679086s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.607922201Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 1.394963896s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.616643415Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 579.725009ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.676613294Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 1.492788694s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.713897879Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 1.425351077s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.72173501Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 1.459921997s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.848972479Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.426012775s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:30.957385172Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 886.943328ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.015127917Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 580.840375ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.040551378Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.094941675s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.161227562Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 722.413293ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.218012417Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 572.821215ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.266524832Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 579.916134ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.382686344Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 1.428381626s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.474169287Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 1.0064227s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.565660406Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 1.292274255s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.604940152Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 889.744394ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.6282643Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 1.333773781s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.639413729Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.479392302s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.670787852Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.236055619s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.68724426Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 1.121680541s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.852231413Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\") failed: Property not found. retry in 1.298073595s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.914204054Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 646.990634ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.925978896Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 625.810842ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.952724454Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 1.49814471s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:31.973367967Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 627.63862ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.015797828Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 570.963759ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.029886457Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 756.33517ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.162202459Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 969.961064ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.222817022Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 534.328239ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.269324406Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 1.003764606s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.323933706Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 541.030696ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.397174678Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 738.464181ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.532076355Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 773.515194ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.587080343Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 907.638198ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.60062787Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 984.050252ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.614956969Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 657.537656ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.627617074Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.012070215s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.654496225Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 612.30819ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.821455049Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 742.315271ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.910926644Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 1.150135862s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.923514742Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 1.110167957s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.935480398Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.075709759s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.951813943Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 1.295554445s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.963959342Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 816.705924ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.984300559Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 964.111498ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:32.996208473Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 768.54675ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.16381972Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 1.369039545s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.207031106Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\") failed: Property not found. retry in 1.067132796s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.228349493Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 761.511221ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.282197787Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 708.048946ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.298013022Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 1.486330919s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.343510605Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 1.079243443s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.374233875Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 592.741719ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.44715098Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 1.05885091s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.484842433Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 1.279209038s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.524055737Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 674.528808ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.615773968Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 1.398590484s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.668309176Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 1.199862196s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.680039646Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 734.765262ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.815367001Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 574.911013ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.943138882Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 1.403366232s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:33.970654596Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 1.477640372s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.017446356Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 512.540127ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.036492816Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.345342935s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.05030446Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 555.928363ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.058902966Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.0325331s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.067195946Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 639.804105ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.081872487Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 948.067548ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.27125307Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 1.469212581s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.300925848Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\") failed: Property not found. retry in 1.024733385s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.310140578Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 895.144548ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.464298349Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.124509243s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.528603658Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 660.510962ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.553547199Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.024776495s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.561585584Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 573.791383ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.5691733Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 616.133647ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.585993865Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 1.320346994s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.624330251Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905\") failed: Property not found. retry in 1.481892126s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.731361366Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 656.666516ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.813601132Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 788.968787ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.846453526Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 928.86361ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:34.890761905Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55\") failed: Property not found. retry in 879.645735ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.084802044Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5\") failed: Property not found. retry in 1.180543536s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.109102203Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 602.721779ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.152979332Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 751.1876ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.212516303Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df\") failed: Property not found. retry in 1.217714163s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.225299665Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7\") failed: Property not found. retry in 664.79217ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.234033534Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\") failed: Property not found. retry in 1.01541272s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.346235625Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 596.141799ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.368715816Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20\") failed: Property not found. retry in 786.905825ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.383978799Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f\") failed: Property not found. retry in 1.200100548s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.40448658Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 528.532947ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.481015879Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8\") failed: Property not found. retry in 806.32037ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.505165604Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be\") failed: Property not found. retry in 1.46401341s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.597159321Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044\") failed: Property not found. retry in 568.209241ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.646127635Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b\") failed: Property not found. retry in 1.408326928s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.663197425Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1\") failed: Property not found. retry in 521.679014ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.768670448Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962\") failed: Property not found. retry in 1.170602179s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.782731377Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511\") failed: Property not found. retry in 585.888202ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.841513642Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb\") failed: Property not found. retry in 890.062456ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.970658371Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e\") failed: Property not found. retry in 1.210186101s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:35.992337295Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97\") failed: Property not found. retry in 1.324959407s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:36.003549295Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6\") failed: Property not found. retry in 741.414895ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:36.021175596Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\") failed: Property not found. retry in 552.949869ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:36.390915489Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294\") failed: Property not found. retry in 609.012115ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:36.963338548Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.628920017Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_cf421c38-9400-403d-b409-822a08e6d3df"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.628989848Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_be1fa9a1-301b-46b9-b30a-f282967efa55"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.628995368Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_clickhouse_39d44799-bf67-4a04-a01e-2dab5d43df6e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.628999688Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_d5f9bf80-883f-4acc-a35d-a7494c2830d6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.629004158Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_b1369ec1-e8a8-48c1-9a6c-96e11e05bc20"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.629008158Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_a21507ee-5c96-4540-bc06-0852e4850905"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.629015358Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_bab6e283-4f89-4430-8daa-991557899d2b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.629020388Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_d3a9225c-107a-4f4b-a8d2-1285aaef40cb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.629041199Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_51231501-67c9-451e-a935-83aa38e09044"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.629053439Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_5b5db754-0dc2-49b5-b4aa-bc8f3c7b0511"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.629057589Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_4b07b723-29d0-44f0-aa06-d84f7872ea97"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.629061389Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_fc451e9c-ecba-4d54-8c4b-8f9b74446294"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.629064799Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_oximeter_24f9cb30-2495-4f1c-8bb5-c4eb0e351bd5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.62906952Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_d17ede40-1516-4fc0-9d4a-6777ee2cc962"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.62907701Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_14ec9e45-b167-4e34-bdf9-2ba3c71c8aa7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.6290829Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_398377fe-8f6c-470f-9364-6ed47bb8001f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.62908791Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_1dd3de1b-db37-4ef3-9d74-b3d6f14ce4be"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.62909278Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_420b93bc-5ca2-4380-8284-8eebea6a5aa1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.62909779Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_2d06eca8-fecb-46e7-b153-2aab6db47cf8"} {"msg":"Writing ledger to /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.629215043Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.632376977Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.636679305Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"f905dc5d-3278-44a9-a9bd-b61e21a8b63d","remote_addr":"[fd00:1122:3344:101::1]:38690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":33033318,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.640338441Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1212"} {"msg":"Writing ledger to /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.640353951Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.640768049Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.643494795Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:679"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.648625701Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:692"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:37.715155405Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:803"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:38.458634752Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:10:38.460252295Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"21.14441299s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:38.561498281Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:38.741111405Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:38.873805746Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:38.965322203Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:39.170820626Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:39.450196806Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:39.727732518Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:40.023308599Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:40.411735734Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:41.056686291Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:41.49016303Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:42.172728967Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:43.148298884Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:43.356440482Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54715"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:43.356515783Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"dff69e22-4e43-4d66-8440-4796a4a0f535","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54715","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":46,"response_code":"200"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:44.539742008Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:46.256736198Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/57b47f3e-5d38-4c51-9327-55cd3ad29f29/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:48.525695959Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dfc06a94-df9b-4b99-b30d-8f102dfd4252\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:10:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dfc06a94-df9b-4b99-b30d-8f102dfd4252\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:50.822691913Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"96134444-e327-4ef2-8dca-0e448c8bd2b7\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:10:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"96134444-e327-4ef2-8dca-0e448c8bd2b7\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:52.217062368Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"616d8cbc-fc90-4111-a69e-78e8c5f48750\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:10:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"616d8cbc-fc90-4111-a69e-78e8c5f48750\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:54.953563346Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b266fc3d-cddb-49ad-aba4-eae49ce7f09e\", \"content-length\": \"124\", \"date\": \"Mon, 29 Apr 2024 04:10:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b266fc3d-cddb-49ad-aba4-eae49ce7f09e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:57.142636028Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:59.738951311Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:59.848189641Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:59906"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:59.848819664Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:59.931682803Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"b7918661-643b-4383-90af-40f20818783c","remote_addr":"[fd00:1122:3344:101::c]:59906","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":83402,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:59.932096292Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:59.932160223Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:59.932613682Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:10:59.932882858Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"65edacfc-55d9-4b8e-a9c6-93ee2d8f50dc","remote_addr":"[fd00:1122:3344:101::c]:59906","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":862,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:02.474017909Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:32952"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:02.475616432Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"222009a4-60e5-4104-8f39-4db965b769f6","remote_addr":"[fd00:1122:3344:101::c]:32952","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":1555,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.383148093Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:52242"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.385575433Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 05866394-eda3-4937-b58c-ca1c4ba1a9ce, kind: Service { id: fc451e9c-ecba-4d54-8c4b-8f9b74446294 }, name: Name(\"external-dns-fc451e9c-ecba-4d54-8c4b-8f9b74446294\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 270759a5-5f64-43fc-97fc-17eb1745bf79, kind: Service { id: 4b07b723-29d0-44f0-aa06-d84f7872ea97 }, name: Name(\"external-dns-4b07b723-29d0-44f0-aa06-d84f7872ea97\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 131, 109])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4e4253d0-a7a0-4801-82ac-dcaca570399b, kind: Service { id: 51231501-67c9-451e-a935-83aa38e09044 }, name: Name(\"nexus-51231501-67c9-451e-a935-83aa38e09044\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 254])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 81961023-ee59-4b62-8c75-fc566ef11bf9, kind: Service { id: a21507ee-5c96-4540-bc06-0852e4850905 }, name: Name(\"nexus-a21507ee-5c96-4540-bc06-0852e4850905\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 154, 143])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: b596142e-278a-430f-8b34-83c69ec604e1, kind: Service { id: be1fa9a1-301b-46b9-b30a-f282967efa55 }, name: Name(\"nexus-be1fa9a1-301b-46b9-b30a-f282967efa55\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 20])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.385631564Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.385676565Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.385687395Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.385708985Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.385727676Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.385736246Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.385767686Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e7bfd52e-af62-4412-8e52-2579c73531e7","remote_addr":"[fd00:1122:3344:101::c]:52242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":2579,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.387568043Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:865"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.632397555Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:56216"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.632492136Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.720311307Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"c4718b08-c6e3-482b-803c-f97a69521447","remote_addr":"[fd00:1122:3344:101::c]:56216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":87828,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.720628174Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.720685495Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.720986551Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:03.721237306Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"e2a0db4d-5cdb-45be-a2be-d888e98fd33d","remote_addr":"[fd00:1122:3344:101::c]:56216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":704,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:13.394950522Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35142"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:13.395065684Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"3bc85b29-6457-4d20-89af-47c898e344e8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35142","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:18.396419981Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:54087"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:18.49192691Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"f9136b50-1737-4756-a406-8c698c535f50","remote_addr":"[fd00:1122:3344:101::b]:54087","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":95455,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:18.492198316Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b1bf7e07-188c-4a0f-b151-9e2544523bd0","remote_addr":"[fd00:1122:3344:101::b]:54087","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":85,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:18.51386559Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:18.51437791Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:18.51582119Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:18.516687978Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:18.516719058Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"PUT","req_id":"b0c7ea8f-129e-4ba8-a087-931d92c5fa74","remote_addr":"[fd00:1122:3344:101::b]:54087","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":3530,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:19.213955607Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:46301"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:19.261621995Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a11ff2c9-5207-4748-9f79-dfd95f01e877","remote_addr":"[fd00:1122:3344:101::a]:46301","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":47614,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:19.2618671Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"0a965f29-dbe1-4c71-85bc-72eeb4a49b69","remote_addr":"[fd00:1122:3344:101::a]:46301","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:19.627464787Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:52115"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:19.672513751Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"7ff65428-859c-4c06-b9f4-70f7756e83b7","remote_addr":"[fd00:1122:3344:101::c]:52115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":44969,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:19.67295027Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"53e1f0cb-b1b8-45b0-ae16-518e8f1195d2","remote_addr":"[fd00:1122:3344:101::c]:52115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":78,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:29.729651992Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:45420"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:29.733633114Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"7f01545f-1bbd-43f0-9d51-7b248df856cf","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":3894,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:43.436558312Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34928"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:43.436646754Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"992b0dcc-f404-404e-b449-2837b5b96bdd","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34928","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":70,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:48.712776256Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:43549"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:48.758554575Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"3805bb2c-0035-4b75-ae36-b5f4a1240490","remote_addr":"[fd00:1122:3344:101::b]:43549","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":45737,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:48.758822321Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"10db8e42-3bc1-4fe3-ad57-f090168faed4","remote_addr":"[fd00:1122:3344:101::b]:43549","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:49.428334671Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47626"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:49.476400017Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0096de94-93cc-42b9-90b3-c1b46ab88b7d","remote_addr":"[fd00:1122:3344:101::a]:47626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":47992,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:49.477475529Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"dd7ef8f2-5003-4f55-a50e-25c0821d8ff2","remote_addr":"[fd00:1122:3344:101::a]:47626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":897,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:49.851623692Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:65240"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:49.895552043Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"50b3f2fd-4f11-4106-b2e7-7d136f162381","remote_addr":"[fd00:1122:3344:101::c]:65240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":43832,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:49.895816958Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f983cfc6-d830-4cc7-9fdc-93086b578542","remote_addr":"[fd00:1122:3344:101::c]:65240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:11:59.730065728Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"dbc5701e-a706-47d9-ae46-2518d19f54b1","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":565,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:13.478646562Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:13.478755115Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"f9b8411e-faf2-49ac-895f-b84511782eda","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57373","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:18.273003685Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:42323"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:18.318447597Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"676b6827-b373-4ffd-b82c-a5bfcf6d8d5c","remote_addr":"[fd00:1122:3344:101::b]:42323","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":45388,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:18.318668452Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"d37825f2-4c4d-496a-9105-42937bf8ce92","remote_addr":"[fd00:1122:3344:101::b]:42323","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:19.20991137Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47369"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:19.258559128Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"74b951a8-efdc-400c-8001-be46b0507048","remote_addr":"[fd00:1122:3344:101::a]:47369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":48592,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:19.258860144Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f9c0b853-91fc-4031-9ba8-984ec4a50abe","remote_addr":"[fd00:1122:3344:101::a]:47369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:19.622523222Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:44981"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:19.67118783Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6073adbe-ac06-4ca1-adf4-b24d8ec1edbb","remote_addr":"[fd00:1122:3344:101::c]:44981","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":48608,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:19.671437585Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"6b572a97-f742-4396-b9c7-3b86bedc724c","remote_addr":"[fd00:1122:3344:101::c]:44981","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:29.73008513Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"a77b04ab-2ffd-48b4-bfcb-53259b5b88ab","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":541,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:43.515454744Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41363"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:43.515565397Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"6eca62fa-49bd-4c94-9a8d-1bde885473c8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41363","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":78,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:48.489856351Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:62370"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:48.53618183Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c4303300-9508-46d3-9db2-f009a6de1047","remote_addr":"[fd00:1122:3344:101::b]:62370","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":46278,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:48.536419406Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"113ba400-6c17-4a7d-8a4c-db424c9e267e","remote_addr":"[fd00:1122:3344:101::b]:62370","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:49.571162726Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:59991"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:49.617857964Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"d1b8a335-1088-4b8f-9c8f-d42397a7c1bb","remote_addr":"[fd00:1122:3344:101::a]:59991","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":46640,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:49.618137119Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"76ecf893-5c1a-4d09-9550-89085c3bdfb2","remote_addr":"[fd00:1122:3344:101::a]:59991","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:49.948328802Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:52932"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:49.99650217Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"9acbc7cc-2e98-441c-bdcc-2b9e1b465e31","remote_addr":"[fd00:1122:3344:101::c]:52932","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":48079,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:49.996747415Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"2f20c762-609a-44d5-89ec-7124831881c3","remote_addr":"[fd00:1122:3344:101::c]:52932","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:12:59.729263661Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"14175198-4b3f-4b7f-af80-da101ebbc2e0","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":856,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:13.55396814Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60011"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:13.554076262Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"e26f47e4-176f-4cbe-97b2-8f491567e6df","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60011","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:18.293875767Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:50684"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:18.338990512Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"5d7b1fb3-1298-4467-ad67-9dbc74e726ed","remote_addr":"[fd00:1122:3344:101::b]:50684","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":45068,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:18.339244338Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"21b51e03-8991-4198-9c1b-4466c705f3c8","remote_addr":"[fd00:1122:3344:101::b]:50684","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:19.421952212Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:37854"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:19.472809585Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"feda9026-53f5-458a-b7db-9ddecf0cc945","remote_addr":"[fd00:1122:3344:101::a]:37854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":50797,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:19.473083251Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7ca7f5e4-57ad-4883-8a27-864e146b407f","remote_addr":"[fd00:1122:3344:101::a]:37854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:19.706629211Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:50607"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:19.754037163Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"d2345ef0-664d-4b86-b4fb-c2af934b63c3","remote_addr":"[fd00:1122:3344:101::c]:50607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":47314,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:19.754336229Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"2fc1c442-53b2-4d77-951e-112b1242f52c","remote_addr":"[fd00:1122:3344:101::c]:50607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:29.734669008Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"ef96eec4-fb5a-485d-9bae-587ae86be674","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":442,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:43.591300663Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49234"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:43.591973647Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"7011ee93-7c6d-4739-a9c4-0a59bf9b835a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49234","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":618,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:48.554822606Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:42985"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:48.600642445Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"eb9d4d27-d54c-42cf-9dcd-183227440481","remote_addr":"[fd00:1122:3344:101::b]:42985","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":45780,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:48.600916291Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7ee1b583-d1f2-4f6d-8bcf-03a4c55f965f","remote_addr":"[fd00:1122:3344:101::b]:42985","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:49.532916355Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:51883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:49.580254836Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"5604cb43-364b-4ffd-954e-c0bcab8e2900","remote_addr":"[fd00:1122:3344:101::a]:51883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":47281,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:49.580492911Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8f20e940-1f22-44bd-a7f2-345c69698b6b","remote_addr":"[fd00:1122:3344:101::a]:51883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:49.731577959Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:47753"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:49.777818758Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c3fabbec-8823-4cdc-ac96-155daf80f8a2","remote_addr":"[fd00:1122:3344:101::c]:47753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":46181,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:49.778050412Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a61f45c1-ac58-4a7f-b7bd-033d944f392e","remote_addr":"[fd00:1122:3344:101::c]:47753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:13:59.795061254Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"13b6153e-1d4d-4663-8bbe-877ccc2e18c8","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":477,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:00.117982746Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:13.623273376Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53005"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:13.623874888Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"9392b39a-98f1-494b-8af0-e5973d0a3c5e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53005","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":58,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:18.307418679Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:32854"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:18.354242959Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"565a489e-5da2-42d3-9712-8c5ca99bea5d","remote_addr":"[fd00:1122:3344:101::b]:32854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":46730,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:18.3547531Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"d5da4641-fb53-4efc-96eb-33bfea67e821","remote_addr":"[fd00:1122:3344:101::b]:32854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":243,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:19.227496818Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40435"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:19.273359808Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"68ecded0-1b20-4a32-9a58-8a4504b167b8","remote_addr":"[fd00:1122:3344:101::a]:40435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":45776,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:19.273602824Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c38f24ac-8133-4212-8546-c65152bf35dc","remote_addr":"[fd00:1122:3344:101::a]:40435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:19.635618808Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:34234"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:19.682066281Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e41bb9f0-92aa-436d-af13-b44082b6a1ad","remote_addr":"[fd00:1122:3344:101::c]:34234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":46389,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:19.682355757Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"e43150be-5c5c-4ad2-ba7c-d7e85ee5496e","remote_addr":"[fd00:1122:3344:101::c]:34234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":112,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:29.793137442Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"c216fdae-3796-4701-922e-a0bcf6ec6077","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":519,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:43.660098306Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43226"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:43.660231529Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"77aba39d-30f3-4dc5-921f-6784142ceefd","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43226","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:48.651543051Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:43006"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:48.696450141Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"8b88927e-4009-43f5-80c7-0df294ac48db","remote_addr":"[fd00:1122:3344:101::b]:43006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":44866,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:48.696701657Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f16385cd-4730-4a38-ae4e-980f183b627e","remote_addr":"[fd00:1122:3344:101::b]:43006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:49.538592282Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63181"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:49.585409602Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e7956b90-33b1-47b8-b29c-9902d4241dbf","remote_addr":"[fd00:1122:3344:101::a]:63181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":46761,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:49.58628121Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8a3fa541-70b0-4da7-8e71-30479620f628","remote_addr":"[fd00:1122:3344:101::a]:63181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":701,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:49.996291749Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:46382"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:50.044118Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"2f84ed02-97ca-482a-9332-ad79e0cb9174","remote_addr":"[fd00:1122:3344:101::c]:46382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":47773,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:50.044369285Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8e6f0def-13f6-4ea3-98a9-60fc8dd246bf","remote_addr":"[fd00:1122:3344:101::c]:46382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":97,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:14:59.793385008Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"a9bb19a8-c65d-48b3-b92c-5d0c6db0cf60","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":1109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:13.700926944Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57459"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:13.701538127Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"51eb246e-9620-404f-bbb6-da101d915f31","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57459","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":67,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:18.287413224Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:60103"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:18.336225145Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"0f93f8cd-5ab8-4d09-8629-b9da04b6f274","remote_addr":"[fd00:1122:3344:101::b]:60103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":48764,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:18.337165984Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f198a712-a381-45a8-be20-a4b1a541b62b","remote_addr":"[fd00:1122:3344:101::b]:60103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":693,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:19.426648628Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58968"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:19.475102841Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b712be63-9c29-4107-a2fa-7c11451a7107","remote_addr":"[fd00:1122:3344:101::a]:58968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":48405,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:19.475389637Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"675e5343-a67c-4237-af3c-0c3ca5f78649","remote_addr":"[fd00:1122:3344:101::a]:58968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:19.623821171Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:47818"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:19.671647772Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"98177cf9-9fea-42ad-a141-a8ecd1d2dad0","remote_addr":"[fd00:1122:3344:101::c]:47818","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":47776,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:19.671986299Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"ebd3903a-80d7-4163-b790-8f999500809e","remote_addr":"[fd00:1122:3344:101::c]:47818","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":137,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:29.793669875Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"3fa8d7bd-91ce-4f7b-a2a9-933bda12b545","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":546,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:43.734268239Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49333"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:43.734837061Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"371bbeb2-d699-4a17-85c4-781b03c8622c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49333","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":563,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:48.528262684Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:38420"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:48.574466912Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"fb7c400f-1006-4770-ae4b-2d6befeb53c1","remote_addr":"[fd00:1122:3344:101::b]:38420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":46159,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:48.574737107Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a85d8b8d-3485-4379-be4a-4eea4222c2c4","remote_addr":"[fd00:1122:3344:101::b]:38420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:49.67636221Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47990"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:49.726470457Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"60156771-f87e-4e9a-a758-dd9ee70dc8b4","remote_addr":"[fd00:1122:3344:101::a]:47990","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":50075,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:49.726736933Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"904e1d3a-8fb9-473c-92f9-f75512819bdc","remote_addr":"[fd00:1122:3344:101::a]:47990","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:49.903535929Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:62270"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:49.950948591Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"46f9525d-8d8d-45d4-b34f-1bfa60818ebe","remote_addr":"[fd00:1122:3344:101::c]:62270","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":47326,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:49.951205506Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"cd900a19-146f-4356-b883-7f866b317e83","remote_addr":"[fd00:1122:3344:101::c]:62270","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:15:59.792804898Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"eab1b8d7-2c62-4e98-8543-92f1f00b85bd","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":493,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:13.772135318Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:13.772793251Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"66e49847-34e7-4041-8005-2e0f7c13a504","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61877","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":631,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:18.375400661Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:53001"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:18.422381845Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a139b661-4fa2-4ccb-89ba-d6753c194455","remote_addr":"[fd00:1122:3344:101::b]:53001","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":46928,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:18.42263363Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"2de3afdc-2cec-49b2-883a-4688746911df","remote_addr":"[fd00:1122:3344:101::b]:53001","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:19.304015275Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:62580"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:19.349064719Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"b938e5e0-5853-405d-affc-ab471a8c7772","remote_addr":"[fd00:1122:3344:101::a]:62580","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":44984,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:19.349736563Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a3bd63bb-0e4e-4a07-ad75-c101e02ab0b5","remote_addr":"[fd00:1122:3344:101::a]:62580","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":523,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:19.664459507Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:33875"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:19.70896616Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"229dba66-4a04-4d7a-a934-9374c7dd60fa","remote_addr":"[fd00:1122:3344:101::c]:33875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":44401,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:19.709308577Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8840c532-c59d-4aa8-a33a-3b5b0605165a","remote_addr":"[fd00:1122:3344:101::c]:33875","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":121,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:29.870116216Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"f1d33800-9609-483b-a13a-e5a6264760ab","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":723,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:43.813792533Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39455"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:43.813912706Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"fc75536e-b194-459c-b2e2-6d1d6f5dc845","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39455","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:48.648645656Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:36137"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:48.698763804Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"9218925c-ba5e-4929-a80b-ec4a6c5c11f5","remote_addr":"[fd00:1122:3344:101::b]:36137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":50057,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:48.699019469Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"49e40a39-1b5e-4987-ad1d-3dc902ddc842","remote_addr":"[fd00:1122:3344:101::b]:36137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:49.58457522Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:42499"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:49.63041889Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e5bbebf0-b0d1-4e12-bf9a-894a614bb821","remote_addr":"[fd00:1122:3344:101::a]:42499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":45778,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:49.630649355Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"c00a5436-475e-418f-9f7e-ccdcd50339e6","remote_addr":"[fd00:1122:3344:101::a]:42499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:50.040318817Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:56475"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:50.087844802Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"2f36e26d-347f-4c07-88b4-945d650593c1","remote_addr":"[fd00:1122:3344:101::c]:56475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":47457,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:50.088080757Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"37f5dedd-d348-473b-a737-83afd2d3eb25","remote_addr":"[fd00:1122:3344:101::c]:56475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":103,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:16:59.870603317Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"849b21f5-fd4a-4baa-80d0-0c9557ec02da","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":1170,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:13.84575265Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64300"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:13.845885363Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"d9d4ee5c-c465-464a-a62e-b599c03fc5da","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64300","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.008597223Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:56429"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.014096886Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:521","metadata":"InstanceMetadata { silo_id: 65cce901-18fa-4cfd-b420-5cdeeed88626, project_id: bcccd9c4-e840-4eca-a8e1-d000028f2138 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-04-29T04:17:17.465778Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(7e2e05ab-2b70-4326-afee-68436500c815), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T04:17:17.520312579Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 3facb129-e543-4212-a83a-33a432ab392b, kind: Instance { id: 415c182f-d43e-459d-a7c3-28d7e56bfa33 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 223, 231])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7583091), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3facb129-e543-4212-a83a-33a432ab392b, kind: Instance { id: 415c182f-d43e-459d-a7c3-28d7e56bfa33 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 223, 231])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7583091), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3facb129-e543-4212-a83a-33a432ab392b, kind: Instance { id: 415c182f-d43e-459d-a7c3-28d7e56bfa33 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 223, 231])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7583091), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(7583091))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3facb129-e543-4212-a83a-33a432ab392b, kind: Instance { id: 415c182f-d43e-459d-a7c3-28d7e56bfa33 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 223, 231])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7583091), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-5aa6c596963\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: ee841b27-c26f-4631-aa06-e6e4957eec08, read_only_parent: Some(Volume { block_size: 512, id: 4b0a2bd8-0a18-4a20-95be-204a7af4dceb, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: d71e19be-664c-4409-b68d-e41088256b92, key: Some(\"0SQbtwRgp+xr5FCCcOg8r5g4fNFXwR+sYwtI6JaaujA=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::15]:19001\", \"[fd00:1122:3344:101::19]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ee841b27-c26f-4631-aa06-e6e4957eec08, key: Some(\"IyOLFQSG4Tzb4pyySeDCMBKr+13F7rTRmf/ejqoFuBI=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAADFKCKdWJ1YAAAoIp1YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAxSginVidWAAAKCKdWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0MTVjMTgyZi1kNDNlLTQ1OWQtYTdjMy0yOGQ3ZTU2YmZhMzMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSDhHanpoWHg0bVU4T2dPOVk0bExGV25DMEtyVEdtODN5cWcvdHBzamt6MCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"7e2e05ab-2b70-4326-afee-68436500c815","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.014197777Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.014569865Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 3facb129-e543-4212-a83a-33a432ab392b, kind: Instance { id: 415c182f-d43e-459d-a7c3-28d7e56bfa33 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 223, 231])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7583091), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3facb129-e543-4212-a83a-33a432ab392b, kind: Instance { id: 415c182f-d43e-459d-a7c3-28d7e56bfa33 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 223, 231])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7583091), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3facb129-e543-4212-a83a-33a432ab392b, kind: Instance { id: 415c182f-d43e-459d-a7c3-28d7e56bfa33 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 223, 231])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7583091), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(7583091))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3facb129-e543-4212-a83a-33a432ab392b, kind: Instance { id: 415c182f-d43e-459d-a7c3-28d7e56bfa33 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 251, 223, 231])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7583091), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-5aa6c596963\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: ee841b27-c26f-4631-aa06-e6e4957eec08, read_only_parent: Some(Volume { block_size: 512, id: 4b0a2bd8-0a18-4a20-95be-204a7af4dceb, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: d71e19be-664c-4409-b68d-e41088256b92, key: Some(\"0SQbtwRgp+xr5FCCcOg8r5g4fNFXwR+sYwtI6JaaujA=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::15]:19001\", \"[fd00:1122:3344:101::19]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ee841b27-c26f-4631-aa06-e6e4957eec08, key: Some(\"IyOLFQSG4Tzb4pyySeDCMBKr+13F7rTRmf/ejqoFuBI=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAADFKCKdWJ1YAAAoIp1YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAxSginVidWAAAKCKdWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0MTVjMTgyZi1kNDNlLTQ1OWQtYTdjMy0yOGQ3ZTU2YmZhMzMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSDhHanpoWHg0bVU4T2dPOVk0bExGV25DMEtyVEdtODN5cWcvdHBzamt6MCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(7e2e05ab-2b70-4326-afee-68436500c815), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T04:17:17.520312579Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-04-29T04:17:17.465778Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"7e2e05ab-2b70-4326-afee-68436500c815","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.014851681Z","hostname":"sock","pid":1590,"uri":"/instances/415c182f-d43e-459d-a7c3-28d7e56bfa33","method":"PUT","req_id":"1e6b87a3-6668-4d97-97ad-ec73b91c0eb2","remote_addr":"[fd00:1122:3344:101::a]:56429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":6202,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.126414239Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53580"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.126856818Z","hostname":"sock","pid":1590,"uri":"/instances/415c182f-d43e-459d-a7c3-28d7e56bfa33/state","method":"PUT","req_id":"58d7a888-1b70-45f5-ad50-eab3790ddbec","remote_addr":"[fd00:1122:3344:101::a]:53580","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":424,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.146578263Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 251, 223, 231]), slot: 0, vni: Vni { inner: 7583091 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.147401829Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.180906106Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.202739723Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.43467964Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:34084"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.481379008Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"59dff2fc-6ddb-44c4-8a90-360ed1027ecb","remote_addr":"[fd00:1122:3344:101::b]:34084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":46602,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:18.481636903Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"51a7499b-f1ee-4590-a166-ba515dce64d6","remote_addr":"[fd00:1122:3344:101::b]:34084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:19.344617441Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:46307"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:19.39136479Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"f93194c9-7d21-4a01-b1de-4421a61d2189","remote_addr":"[fd00:1122:3344:101::a]:46307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":46668,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:19.391740167Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"76fe1806-31fe-4868-b1f3-b17e796b9c0c","remote_addr":"[fd00:1122:3344:101::a]:46307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:19.702699365Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:35618"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:19.746079915Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"d7076b6a-94cc-410c-9543-94eb5d41f85f","remote_addr":"[fd00:1122:3344:101::c]:35618","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":43306,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:19.74635058Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"fac38dfb-109d-4f53-bb6c-309cc114a8b2","remote_addr":"[fd00:1122:3344:101::c]:35618","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":107,"response_code":"200"} {"msg":"Profile for oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:19.786728108Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:19.786936852Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815\") failed: Property not found. retry in 27.930987ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:17:19.981329489Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815\") failed: Property not found. retry in 61.704578ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:17:20.019185356Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815\") failed: Property not found. retry in 108.889504ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:17:20.099327599Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815\") failed: Property not found. retry in 433.430011ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:17:20.227802504Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815\") failed: Property not found. retry in 494.87157ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:17:20.672088025Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815\") failed: Property not found. retry in 1.000293185s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:17:21.180756347Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815\") failed: Property not found. retry in 1.226092197s","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:17:22.190700259Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815\") failed: Property not found. retry in 902.83469ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:17:23.427834Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815\") failed: Property not found. retry in 937.321222ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:17:24.340823683Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:25.304316953Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:25.312997951Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:25.345910816Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAADFKCKdWJ1YAAAoIp1YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAxSginVidWAAAKCKdWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI0MTVjMTgyZi1kNDNlLTQ1OWQtYTdjMy0yOGQ3ZTU2YmZhMzMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSDhHanpoWHg0bVU4T2dPOVk0bExGV25DMEtyVEdtODN5cWcvdHBzamt6MCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-5aa6c596963\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: ee841b27-c26f-4631-aa06-e6e4957eec08, read_only_parent: Some(Volume { block_size: 512, id: 4b0a2bd8-0a18-4a20-95be-204a7af4dceb, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: d71e19be-664c-4409-b68d-e41088256b92, key: Some(\"0SQbtwRgp+xr5FCCcOg8r5g4fNFXwR+sYwtI6JaaujA=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::15]:19001\", \"[fd00:1122:3344:101::19]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: ee841b27-c26f-4631-aa06-e6e4957eec08, key: Some(\"IyOLFQSG4Tzb4pyySeDCMBKr+13F7rTRmf/ejqoFuBI=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::13]:19000\", \"[fd00:1122:3344:101::12]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 415c182f-d43e-459d-a7c3-28d7e56bfa33, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: bcccd9c4-e840-4eca-a8e1-d000028f2138, silo_id: 65cce901-18fa-4cfd-b420-5cdeeed88626 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:25.349585061Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:25.461154399Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-04-29T04:17:25.461678330Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:25.461693381Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:25.461701061Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-04-29T04:17:25.461678330Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7e2e05ab-2b70-4326-afee-68436500c815), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T04:17:17.520312579Z }","propolis_id":"7e2e05ab-2b70-4326-afee-68436500c815"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:25.461719881Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7e2e05ab-2b70-4326-afee-68436500c815), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T04:17:17.520312579Z }, propolis_id: 7e2e05ab-2b70-4326-afee-68436500c815, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-04-29T04:17:25.461678330Z } }","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-04-29T04:17:25.520720111Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:25.520745021Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:25.520795793Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-04-29T04:17:25.520720111Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7e2e05ab-2b70-4326-afee-68436500c815), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T04:17:17.520312579Z }","propolis_id":"7e2e05ab-2b70-4326-afee-68436500c815"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:25.520806713Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7e2e05ab-2b70-4326-afee-68436500c815), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T04:17:17.520312579Z }, propolis_id: 7e2e05ab-2b70-4326-afee-68436500c815, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-04-29T04:17:25.520720111Z } }","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:29.870961146Z","hostname":"sock","pid":1590,"uri":"//94bd4353-ea99-483d-acc9-6db1c4f75870","method":"GET","req_id":"161581b9-fa02-4dfe-b639-7634aca67087","remote_addr":"[fd00:1122:3344:101::d]:45420","local_addr":"[fd00:1122:3344:101::1]:48068","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"94bd4353-ea99-483d-acc9-6db1c4f75870","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":1309,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:35.705967291Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63109"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:35.707079303Z","hostname":"sock","pid":1590,"uri":"/instances/415c182f-d43e-459d-a7c3-28d7e56bfa33/state","method":"PUT","req_id":"e38a470e-216b-46a8-ad1e-9b72e6f3375c","remote_addr":"[fd00:1122:3344:101::a]:63109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":1086,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-04-29T04:17:35.707181425Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:35.707205966Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:35.707245347Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-04-29T04:17:35.707181425Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7e2e05ab-2b70-4326-afee-68436500c815), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T04:17:17.520312579Z }","propolis_id":"7e2e05ab-2b70-4326-afee-68436500c815"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:35.707263137Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7e2e05ab-2b70-4326-afee-68436500c815), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-29T04:17:17.520312579Z }, propolis_id: 7e2e05ab-2b70-4326-afee-68436500c815, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-04-29T04:17:35.707181425Z } }","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-04-29T04:17:35.748200317Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:35.748224447Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:35.748249418Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-04-29T04:17:35.748200317Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-29T04:17:35.748200317Z }","propolis_id":"7e2e05ab-2b70-4326-afee-68436500c815"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:35.748275968Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:35.748283448Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:35.749705608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:35.749758229Z","hostname":"sock","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/50faa4b2-c048-4a8a-9779-53ba148e2f90/debug/bundle/zone\", \"/pool/int/659cec67-f7a4-40fc-9bb6-e073575516b2/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/d1b1e109-515b-482d-930d-ba493cd78ef0/crypt/debug\", \"/pool/ext/7b71d6bd-a486-4799-a3b4-23ca97ac0816/crypt/debug\", \"/pool/ext/a1b82a77-810b-4e64-9e93-3bddcb08ae61/crypt/debug\", \"/pool/ext/24a77df0-4435-4add-9951-0211a2cf4c49/crypt/debug\", \"/pool/ext/6686ffc9-9b48-48c1-988b-7aac0a49e5a5/crypt/debug\", \"/pool/ext/92f3b999-4455-49cb-8ded-aa658c2709d6/crypt/debug\", \"/pool/ext/feef97bb-ca23-472f-b173-ce352fd7d46c/crypt/debug\", \"/pool/ext/9ffa1733-2919-4d64-8165-6d72c0607851/crypt/debug\", \"/pool/ext/05c910b1-d5d8-4b62-8b1a-0afed5d62eeb/crypt/debug\"] }","zone_name":"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:36.375173135Z","hostname":"sock","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815\", bundle_id: 5c009c80-dda4-4621-b060-3153cdb9985b }, time_created: 2024-04-29T04:17:35.749894441Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_7e2e05ab-2b70-4326-afee-68436500c815","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:17:36.375275286Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:38.126745647Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:38.15032554Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-29T04:17:35.748200317Z }, propolis_id: 7e2e05ab-2b70-4326-afee-68436500c815, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-04-29T04:17:38.150236858Z } }","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:38.223525621Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:52202"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-04-29T04:17:38.224428959Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:38.224444349Z","hostname":"sock","pid":1590,"uri":"/v2p/3facb129-e543-4212-a83a-33a432ab392b","method":"DELETE","req_id":"b3793020-8e69-4adb-b923-370e6ba6a2d2","remote_addr":"[fd00:1122:3344:101::c]:52202","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-40c91ce0f6fb1682/b5647d2/dropshot/src/server.rs:849","latency_us":838,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:38.526265159Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-29T04:17:35.748200317Z }, propolis_id: 7e2e05ab-2b70-4326-afee-68436500c815, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-04-29T04:17:38.150236858Z } }","instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-04-29T04:17:38.530272582Z","hostname":"sock","pid":1590,"instance_id":"415c182f-d43e-459d-a7c3-28d7e56bfa33","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"}