[ Apr 30 22:19:16 Enabled. ] [ Apr 30 22:19:16 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Apr 30 22:19:16 Method "start" exited with status 0. ] [ Apr 30 22:19:16 Rereading configuration. ] [ Apr 30 22:19:16 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:16.528144669Z","hostname":"sock","pid":1590,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:16.528289902Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:16.582820118Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:16.582862899Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:16.698107916Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:16.706691473Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:16.714606442Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.313964402Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.456829766Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.456872767Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.45698049Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.457125904Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.457164405Z","hostname":"sock","pid":1590,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:291"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.457174505Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.457232847Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.460387046Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:17.460399146Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.460420157Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.460423757Z","hostname":"sock","pid":1590,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.460445188Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.460503279Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.46055375Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.46056464Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.460597761Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.472663766Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.472699337Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.572661019Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.780566343Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.780601314Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.780606704Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.780618985Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.780624445Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.780628315Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.780647625Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.789366145Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.789400576Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:17.877246472Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.079698819Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.07973922Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.079811502Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.079823353Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.079863174Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.079903425Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.079920515Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.079924495Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.079951696Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.079984507Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080010387Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080014098Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080019087Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080023578Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080038078Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080041568Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080047108Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080077119Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080081659Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.08009742Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.08010194Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.08012718Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080142001Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080146741Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080163451Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080167961Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080172731Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080176951Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080193992Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080218523Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080238813Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080256933Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080261564Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080267154Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080270424Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080274694Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080301305Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080310885Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080319425Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080339806Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080362856Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080425008Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080446638Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080454049Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080476899Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.080483559Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.08049994Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.08051147Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.08051989Z","hostname":"sock","pid":1590,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.091394254Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.091432215Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999929608s","next_cleanup":"Instant { tv_sec: 5001, tv_nsec: 855272792 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.091458216Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.091505217Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.09439401Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.094478072Z","hostname":"sock","pid":1590,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.094829551Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.094910653Z","hostname":"sock","pid":1590,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV1 {\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV1 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n },\n ],\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.09517856Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:979"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.09518703Z","hostname":"sock","pid":1590,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095248372Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095289073Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095303773Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095314633Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1039"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095343064Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095357914Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095372215Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095379815Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1080"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095385195Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095390075Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095420886Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095436887Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.095738084Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.09597948Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.09599325Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1113"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096038812Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096128984Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096329179Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096469273Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096502533Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 03a52263-53ab-4221-9c03-9637ab9d1295, rack_id: cbc2e9a2-a4c8-47b8-a5d7-5151055087a9, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096705048Z","hostname":"sock","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:35146","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 03a52263-53ab-4221-9c03-9637ab9d1295, rack_id: cbc2e9a2-a4c8-47b8-a5d7-5151055087a9, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096806671Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096812961Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096842622Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096846922Z","hostname":"sock","pid":1590,"file":"sled-agent/src/server.rs:44"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096867783Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096871423Z","hostname":"sock","pid":1590,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"No ledger in /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096878053Z","hostname":"sock","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096881903Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096895443Z","hostname":"sock","pid":1590,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096900824Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.096904113Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.099598401Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.107377238Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.117513553Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.128240314Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.138351779Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.147227693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.156104757Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.165195686Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.173924667Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.183077787Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.183134309Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.183139429Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.183143759Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.183147679Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.183151569Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.204105228Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.221601349Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.233295514Z","hostname":"sock","pid":1590,"sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:352"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.233382007Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.233391637Z","hostname":"sock","pid":1590,"sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:364"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.233401627Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 56.011197ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.236334421Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2d1c2b20-bcef-49ec-bd76-03cf0170634a\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:19:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2d1c2b20-bcef-49ec-bd76-03cf0170634a\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_8a27d52c-0cf2-44eb-a59a-d16436f25542/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.247828931Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_8a27d52c-0cf2-44eb-a59a-d16436f25542/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.271953029Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_8a27d52c-0cf2-44eb-a59a-d16436f25542/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.27199433Z","hostname":"sock","pid":1590,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.277073089Z","hostname":"sock","pid":1590,"sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:379"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.277081599Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_8a27d52c-0cf2-44eb-a59a-d16436f25542/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.277090609Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.293674137Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 167.298645ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.294204531Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3f4053fd-193b-4dbd-b275-137e998a416f\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:19:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3f4053fd-193b-4dbd-b275-137e998a416f\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.345947616Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.46224092Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 121.0317ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.462851976Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f491d2d6-a0bc-464b-a662-432a0308082a\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:19:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f491d2d6-a0bc-464b-a662-432a0308082a\" }"} {"msg":"Mounting oxi_8a27d52c-0cf2-44eb-a59a-d16436f25542/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.564771687Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.577732843Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.577824666Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.577855067Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.577884297Z","hostname":"sock","pid":1590,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.584833563Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 156.652036ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.585344546Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"22c7b1e9-c05b-499b-af9c-bb1c72f8650a\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:19:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"22c7b1e9-c05b-499b-af9c-bb1c72f8650a\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.743395123Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 157.306948ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.743738611Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e92487aa-96e2-410c-b10f-25617dbbd716\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:19:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e92487aa-96e2-410c-b10f-25617dbbd716\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.902679821Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 188.855291ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:18.903597734Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a1811ba0-2e44-41e6-abf0-3a55d53c48db\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:19:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a1811ba0-2e44-41e6-abf0-3a55d53c48db\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.09331058Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 403.888355ms","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.093966866Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4822c6b4-c5b0-4b19-88f6-9ea156ffc276\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:19:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4822c6b4-c5b0-4b19-88f6-9ea156ffc276\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.288773511Z","hostname":"sock","pid":1590,"sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.289458018Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.289486779Z","hostname":"sock","pid":1590,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.289597141Z","hostname":"sock","pid":1590,"file":"oximeter/producer/src/lib.rs:284","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:284","interval":"30s","address":"[fd00:1122:3344:101::1]:35283","producer_id":"03a52263-53ab-4221-9c03-9637ab9d1295","route":"//03a52263-53ab-4221-9c03-9637ab9d1295"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:19.297163342Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"360.904905ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.338106325Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.338149196Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.338226818Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.338235279Z","hostname":"sock","pid":1590,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.338930766Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.338967657Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.338990578Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.339004388Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.339268765Z","hostname":"sock","pid":1590,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.339456879Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.33949849Z","hostname":"sock","pid":1590,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100 }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.339522541Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.339543001Z","hostname":"sock","pid":1590,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:35146","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.339563232Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.339573262Z","hostname":"sock","pid":1590,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.512603557Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.513549201Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.513647514Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.513654974Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.513693495Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.513706055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:19.518523947Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.545835036Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35491"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.54598742Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.546078302Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"16c52d24-f8d5-4f74-88d8-174562b873c2","remote_addr":"[fd00:1122:3344:101::1]:35491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.57536263Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:34205"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.575464793Z","hostname":"sock","pid":1590,"uri":"/sled-role","method":"GET","req_id":"0feb79ef-431a-47fe-a36d-dfc76a790cda","remote_addr":"[fd00:1122:3344:101::1]:34205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":37,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.576869608Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.57690432Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.577175216Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.577371601Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 5f639cfb-227e-47bf-96f7-d9691e1d7c0c, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool) }, OmicronPhysicalDiskConfig { id: 13c73dda-a8e3-471f-92cd-ef10f91214fd, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool) }, OmicronPhysicalDiskConfig { id: 9799720f-ce7e-47e2-b86f-deaaf2df7ed2, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool) }, OmicronPhysicalDiskConfig { id: c27f5bb2-7fce-43c1-a377-ddd89826d17e, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool) }, OmicronPhysicalDiskConfig { id: 5338df58-3bb2-4ee7-b487-73524b99b369, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool) }, OmicronPhysicalDiskConfig { id: d32fc17d-df26-4fa8-8b17-435f93136719, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool) }, OmicronPhysicalDiskConfig { id: bcf027a6-cf5a-4793-83b2-39882d01fca8, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool) }, OmicronPhysicalDiskConfig { id: a9c9da13-c2b7-4337-aa03-1239b4679802, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool) }, OmicronPhysicalDiskConfig { id: f080f886-8d91-4ac6-b9a1-b1b64b373679, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.608409244Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:346"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.608489176Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:61573"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: 5f639cfb-227e-47bf-96f7-d9691e1d7c0c, pool_id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: 13c73dda-a8e3-471f-92cd-ef10f91214fd, pool_id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: 9799720f-ce7e-47e2-b86f-deaaf2df7ed2, pool_id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: c27f5bb2-7fce-43c1-a377-ddd89826d17e, pool_id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: 5338df58-3bb2-4ee7-b487-73524b99b369, pool_id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: d32fc17d-df26-4fa8-8b17-435f93136719, pool_id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: bcf027a6-cf5a-4793-83b2-39882d01fca8, pool_id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: a9c9da13-c2b7-4337-aa03-1239b4679802, pool_id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: f080f886-8d91-4ac6-b9a1-b1b64b373679, pool_id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.608683871Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.608689631Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.608698242Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.608702652Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.608719382Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.608727072Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.608731902Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.608735883Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:19.691694565Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"577.755052ms"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.697861341Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:19.697871481Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:20.496002696Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.140259945s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-30T22:19:20.496414316Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:20.690831181Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:20.863789334Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:20.863902737Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:20.863927208Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:20.863934218Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:20.863941978Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:20.864074781Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:20.864081931Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:21.398749989Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:21.400620616Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-30T22:19:21.636256781Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:21.637828591Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.247102591s"} {"msg":"Automatically destroying dataset: oxp_9a1451e3-b124-44f6-8078-fbc416821fc9/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:21.641105603Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:21.988274621Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 29.2421ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:22.728930366Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 98.918365ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:23.2214202Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-30T22:19:23.480229529Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:23.480554927Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 258.929509ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:23.76329924Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:24.233110282Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.211499737s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 468.810194ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:24.452841735Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-30T22:19:24.696800909Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 443.359316ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:25.177521556Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Looking for unencrypted datasets in oxp_9a1451e3-b124-44f6-8078-fbc416821fc9","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:25.191379116Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:25.401363893Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:25.401455676Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:25.401469086Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:25.401479316Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:25.401486106Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:25.628786911Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:25.628813621Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"52a73ea1-1580-48fa-9393-5bb705994a44 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:26.794271222Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:27.047203613Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:27.047239743Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:27.047284675Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:27.047319375Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:27.047329116Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:27.047447319Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:27.047464979Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-30T22:19:28.040296586Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:28.2937814Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:28.523587378Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:28.760287319Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.935334433s"} {"msg":"Automatically destroying dataset: oxp_52a73ea1-1580-48fa-9393-5bb705994a44/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:29.033255075Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-30T22:19:29.523916373Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:29.767117028Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:29.768480723Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:29.989805666Z","hostname":"sock","pid":1590,"component":"VmmReservoirManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Looking for unencrypted datasets in oxp_52a73ea1-1580-48fa-9393-5bb705994a44","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.105758911Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.114615724Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.114656285Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.114664915Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.114670785Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.114685906Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.124457582Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.124478373Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"03d465fc-bc73-4466-af31-1a5670be772e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.216843053Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.223921602Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.223969963Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.223977613Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.224009484Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.224030314Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.224187318Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.224193968Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.2571577Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_03d465fc-bc73-4466-af31-1a5670be772e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.26506031Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-30T22:19:30.275673797Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_03d465fc-bc73-4466-af31-1a5670be772e","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.377715381Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.38479827Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.384846161Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.384852321Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.384859742Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.384864202Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.395165412Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.395212103Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.486399683Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.49420028Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.494235811Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.494257132Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.494296122Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.494323983Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.494412315Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.494421896Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.530581718Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_145ad0b0-5e5c-4755-ba6c-da149e549070/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.538633381Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_145ad0b0-5e5c-4755-ba6c-da149e549070","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.666494797Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.675027682Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.675098894Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.675108504Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.675115834Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.675123134Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.685057045Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.685089236Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.778053381Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.78634509Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.786377731Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.786385441Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.786391901Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.786402322Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.786533945Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.786555826Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.821723213Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b110b23f-2145-46b8-8de0-dc91bc41dd55/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.829716094Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_b110b23f-2145-46b8-8de0-dc91bc41dd55","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.953850226Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.962441843Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.962491944Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.962502854Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.962510974Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.962537635Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.972464865Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:30.972504246Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.072287923Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.08086186Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.080900941Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.080909311Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.080978403Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.080992143Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.081113936Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.081121516Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.117463443Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d3fbf4c0-71ec-480f-942f-6fd5fd537b7c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.125691021Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_d3fbf4c0-71ec-480f-942f-6fd5fd537b7c","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.252068719Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.259931797Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.259980618Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.259987709Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.259995819Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.260002379Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.269803076Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.269832057Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.366411994Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.374799785Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.374839006Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.374846246Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.374879347Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.374892548Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.375038061Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.375047701Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-30T22:19:31.37897712Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:31.379181666Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.409391678Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c8e411fe-43b9-47e1-88c5-1303772b3e4e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.417830061Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_c8e411fe-43b9-47e1-88c5-1303772b3e4e","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.543812819Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.552508638Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.552553349Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.55256077Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.55256812Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.552607611Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.562387847Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.562418088Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"e10de419-fa2f-479e-b380-a88997c87dc2 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.666446323Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.675242034Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.675289986Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.675300396Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.675309216Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.675317876Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.67544591Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.67545536Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.710896814Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e10de419-fa2f-479e-b380-a88997c87dc2/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.718143627Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_e10de419-fa2f-479e-b380-a88997c87dc2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.835099897Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.843089369Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.84314304Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.843154621Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.843163421Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.843183811Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.853384179Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.85342404Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.951536325Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.959637889Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.959706021Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.959713511Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.959747372Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.959761152Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.959924566Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.959933147Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:31.997256988Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_bb74811e-acf4-486b-913b-bc09ce8f6984/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.006042729Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_bb74811e-acf4-486b-913b-bc09ce8f6984","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.138133631Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.146935924Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.146984595Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Writing ledger to /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.147013446Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.14717555Z","hostname":"sock","pid":1590,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 8a27d52c-0cf2-44eb-a59a-d16436f25542 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c953fe1d-1619-477a-bac7-ceb27e5a468a (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Writing ledger to /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.147308913Z","hostname":"sock","pid":1590,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.147662662Z","hostname":"sock","pid":1590,"uri":"/omicron-physical-disks","method":"PUT","req_id":"bd9222ad-bfd9-420e-8d00-0c5c395ae4b8","remote_addr":"[fd00:1122:3344:101::1]:61573","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12539113,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.178195592Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.178343476Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58552"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.178651093Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.178660164Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.228595524Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.228633625Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.2811906Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.281222071Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.332617888Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.33269274Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.33271626Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.332765812Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.380552087Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.380652269Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.38066307Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-30T22:19:32.384813954Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.435451972Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.439551665Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.443669689Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.454701028Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.460044612Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:32.464012952Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:33.37764124Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-04-30T22:19:33.387322745Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:34.621465599Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:34.621598632Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:34.621773617Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:35.105178481Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:35.58518627Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:35.585329914Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:35.585336604Z","hostname":"sock","pid":1590,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1\") failed: Property not found. retry in 71.341024ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:35.593239684Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:35.593280425Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:35.597537422Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1\") failed: Property not found. retry in 108.104247ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:35.674273008Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8\") failed: Property not found. retry in 61.385363ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:35.725349437Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08\") failed: Property not found. retry in 46.748966ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:35.731673486Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08\") failed: Property not found. retry in 74.184245ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:35.78573917Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1\") failed: Property not found. retry in 120.00869ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:35.803057937Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8\") failed: Property not found. retry in 77.840564ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:35.912508268Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08\") failed: Property not found. retry in 269.959977ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:35.93559759Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1\") failed: Property not found. retry in 476.160188ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:35.966291585Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8\") failed: Property not found. retry in 106.903573ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:36.009075214Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8\") failed: Property not found. retry in 430.640026ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:36.135290388Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08\") failed: Property not found. retry in 225.953733ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:36.218434665Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1\") failed: Property not found. retry in 477.654433ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:36.452724106Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08\") failed: Property not found. retry in 428.702171ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:36.459965449Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8\") failed: Property not found. retry in 578.530475ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:36.57538114Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08\") failed: Property not found. retry in 723.873471ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:36.900901412Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1\") failed: Property not found. retry in 784.728279ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:36.941068346Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8\") failed: Property not found. retry in 742.59186ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:37.16560559Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08\") failed: Property not found. retry in 769.082628ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:37.634575581Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:37.663304095Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:37.663323486Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1\") failed: Property not found. retry in 1.153507612s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:37.736002159Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8\") failed: Property not found. retry in 1.178578978s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:37.916422361Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe7a:4c01","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:38.21768062Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:38.223730253Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:38.343964116Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"} {"msg":"wait for service Some(\"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08\") failed: Property not found. retry in 657.386354ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:38.413599093Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:38.590028404Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:38.779788641Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"} {"msg":"wait for service Some(\"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1\") failed: Property not found. retry in 972.995771ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:38.897856089Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:39.075119871Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"} {"msg":"wait for service Some(\"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08\") failed: Property not found. retry in 1.451475906s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:39.080914197Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8\") failed: Property not found. retry in 1.340937342s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:39.103782944Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:39.281098247Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:39.459649672Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:39.677130558Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:39.854944064Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:39.865456909Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:39.86549711Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:39.86550684Z","hostname":"sock","pid":1590,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1\") failed: Property not found. retry in 1.044862929s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:39.881657158Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:40.417949207Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:40.423728612Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"wait for service Some(\"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8\") failed: Property not found. retry in 752.226949ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:40.454704494Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08\") failed: Property not found. retry in 843.935823ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:40.545043393Z","hostname":"sock","pid":1590,"zone":"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:40.574083036Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:40.688777179Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:40.714885848Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:40.836838774Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:40.984419927Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:364"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.016964148Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:412"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.048963515Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:589"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.049050317Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:627"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.050249457Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:630","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.050258068Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:434","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128628954Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128710387Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128739597Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128747828Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128753738Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128761218Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128767378Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128772788Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128778258Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128782389Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128788149Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128794209Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128800159Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128805829Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128816389Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.128822239Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.12882814Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.12883357Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.12883764Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.1288418Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.12884567Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.1288498Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.12885368Z","hostname":"sock","pid":1590,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.407340146Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_73cb0d1c-76b9-46e7-a3c4-339faca846c1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.407413378Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_fc6accb5-93bc-4f13-b90c-1fe0dce9aa08"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.407423948Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_070282d1-0314-4f24-9b9b-f3fe2fd417c8"} {"msg":"Writing ledger to /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.407463899Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.407769116Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.408018863Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"1159f209-1322-4258-bfcd-905590386693","remote_addr":"[fd00:1122:3344:101::1]:58552","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9229598,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.410022193Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"failed to write DNS configuration (will retry in 105.93206ms)","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:41.439956999Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:567","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.563243709Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.607942077Z","hostname":"sock","pid":1590,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:547"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.659964779Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:583"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.659976669Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.694186612Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.695181847Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.695188147Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.724008054Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.724108767Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:37559"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.724277511Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.724286452Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.746525113Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.746570224Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.765652385Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.765684496Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.784723966Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.784772297Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.78488647Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.784973223Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.796289368Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.827976467Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.830316666Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.830333227Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.876319387Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 149, 130]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.876349088Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.877126357Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.877134888Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.877479026Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.878894092Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.900973709Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:41.919969398Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:43.056001477Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:43.056197061Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d\") failed: Property not found. retry in 67.861854ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:43.188096479Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d\") failed: Property not found. retry in 91.045206ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:43.27016839Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d\") failed: Property not found. retry in 177.144946ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:43.391352897Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d\") failed: Property not found. retry in 353.157322ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:43.578896638Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d\") failed: Property not found. retry in 1.026191069s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:43.941840484Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:44.342144812Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35316"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:44.342408189Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"b0b54cc0-683d-4fc9-8b7c-f54d6b7e9794","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35316","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":127,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d\") failed: Property not found. retry in 764.121532ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:44.979775838Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d\") failed: Property not found. retry in 1.243692332s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:45.753308532Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d\") failed: Property not found. retry in 1.286916973s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:47.007623784Z","hostname":"sock","pid":1590,"zone":"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:48.314981185Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d"} {"msg":"Writing ledger to /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:48.315050706Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:48.315374495Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:48.315641681Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"eda7746a-0af0-41f9-8c32-0f4d02c8a501","remote_addr":"[fd00:1122:3344:101::1]:37559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6591490,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:48.317804226Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:624"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:48.346541581Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:48.346694375Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35855"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:48.35798684Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:48.358127393Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"19b4ab8f-6dc2-44b4-9acb-82159231c41b","remote_addr":"[fd00:1122:3344:101::1]:35855","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":11290,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:48.361528449Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:48.49157134Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:48.491679862Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51461"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:52.501270333Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:52.501417436Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"9d9988c3-1fd6-43c4-827d-f87f84e190d4","remote_addr":"[fd00:1122:3344:101::1]:51461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009559,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:52.505471879Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:52.614703454Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:52.614842458Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48354"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:52.623847055Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:52.623963038Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"592928bf-761f-4e3a-ba27-8e24ca9e5f54","remote_addr":"[fd00:1122:3344:101::1]:48354","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8998,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:52.626588594Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:52.849285172Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:52.849384025Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52377"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_fc04604b-21ab-4a40-a700-a181ed4f060d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:52.858643248Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:52.858758211Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"c1bac45e-f364-4beb-8389-7d739651a570","remote_addr":"[fd00:1122:3344:101::1]:52377","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9249,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:52.860446004Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:53.139644596Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:53.139762779Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:57578"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:54.149641086Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"a5894497-25b7-487f-98cd-e0a5b6c87387","remote_addr":"[fd00:1122:3344:101::1]:57578","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1009722,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714515592.146331, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:54.152820046Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:54.152837956Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:54.369277156Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:54.36943422Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52614"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:54.378335185Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"73b12518-a837-4957-beb5-1bb5e0de71b8","remote_addr":"[fd00:1122:3344:101::1]:52614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8771,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714515592.146331, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:54.382227833Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:54.382235914Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:54.783117997Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:54.783230679Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:61170"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:54.791083418Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"ebf5b10f-d618-498e-b5cb-0603ccdd6e9a","remote_addr":"[fd00:1122:3344:101::1]:61170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7704,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714515592.146331, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:54.794845403Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:54.794854243Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:55.234062683Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:55.234168305Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:58883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:55.241703835Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"b291f52b-f405-4eaf-84af-0be95534c420","remote_addr":"[fd00:1122:3344:101::1]:58883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7410,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714515592.146331, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:55.243756057Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:55.243761997Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:55.729660175Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:55.729771388Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:52042"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:55.739075763Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"23533579-b2e6-4045-8060-94aeadaaeebc","remote_addr":"[fd00:1122:3344:101::1]:52042","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9133,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714515592.146331, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:55.742184401Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:55.742192221Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:56.205841558Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:56.20594336Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38192"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:56.215183323Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"9a2cc18f-ab0b-4ee7-94a7-6b60c25539b1","remote_addr":"[fd00:1122:3344:101::1]:38192","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9109,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714515592.146331, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:56.217638585Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:56.217646895Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:56.822990136Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:56.82310235Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:39136"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:56.831536382Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"0d61de89-b5c6-4c9d-8eb0-c84fe197f610","remote_addr":"[fd00:1122:3344:101::1]:39136","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8253,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714515592.146331, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:56.834761204Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:56.834769294Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:57.718758124Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:57.718860937Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48557"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:57.728315395Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"5da063eb-566b-4940-9ac8-3640b47adb7c","remote_addr":"[fd00:1122:3344:101::1]:48557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9296,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1714515592.146331, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:57.73009849Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:57.73010654Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:654","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.057348126Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:604"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.057454499Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:55357"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.060450234Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.077966156Z","hostname":"sock","pid":1590,"uri":"/timesync","method":"GET","req_id":"9129b87e-238b-4dc5-83a4-53323b2c1c98","remote_addr":"[fd00:1122:3344:101::1]:55357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":20338,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2491897134, ip_addr: 148.135.85.46, stratum: 3, ref_time: 1714515597.730142, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.080155312Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.080169662Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1192"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.108743583Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.108861736Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:63723"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.109102462Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.109108612Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.13084903Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.130903362Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.151061451Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.151110552Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.169215558Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.16925715Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.221656401Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.221705422Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.274371401Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.274413032Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.326102616Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.326147897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.379965955Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.380020536Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.433172417Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.433228159Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.433444534Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.433555157Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.518629343Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.518678054Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.518684354Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.518690255Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.518697645Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.611849285Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.616151733Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.620489823Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.624961946Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.629643354Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.634184528Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.638873767Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.643331649Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.647808882Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:19:58.652242804Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:19:59.53130428Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"188.207771ms"} {"msg":"Profile for oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:02.06882647Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:02.068973423Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:02.072326068Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"Profile for oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:02.111449275Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:02.111588829Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:02.154118142Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:02.154297496Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:02.160302407Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:02.160450871Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 32.835108ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.222970438Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 56.308762ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.263525011Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 110.736858ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.271979545Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:02.276995111Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:02.277112034Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 64.601858ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.316155049Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 57.128862ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.329075755Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 78.030266ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.341623542Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 95.742143ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.46840299Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 191.494548ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.515278742Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 86.688475ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.596079351Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 232.566137ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.733876687Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 219.873749ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.810514161Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 468.827377ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.843435201Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 236.445159ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.920017363Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 30.224623ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:02.983045703Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 595.588528ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:03.108068177Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 366.76084ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:03.255079085Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 344.329173ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:03.35712398Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 124.584165ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:03.461474482Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 807.73803ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:03.506360424Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 208.343734ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:03.606496841Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 807.315337ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:03.644419598Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 786.108786ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:03.754876944Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 885.770795ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:03.825240188Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 421.520763ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:03.88952185Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 549.269511ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:04.374925386Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 1.268754939s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:04.404497272Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 1.341300442s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:04.516643941Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 1.396433038s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:04.569057693Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 1.364283465s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:04.722177546Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 621.75521ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:04.937309133Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 848.036611ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:05.570064366Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 913.856927ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:05.683376914Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 1.215311069s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:05.868690899Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 826.869495ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:05.974482827Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 1.090714019s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:06.096556547Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 850.603596ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:06.428628975Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 1.172716303s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:06.608202435Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 1.354024411s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:06.812350855Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 1.069470555s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:07.09717439Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 562.182335ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:07.202283801Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 1.437694315s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:07.292408245Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 1.210429568s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:07.775042031Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 1.480517281s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:07.789552027Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 917.242289ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:08.175526864Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 633.919621ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:08.183406532Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 795.570407ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:08.738885076Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 1.240496782s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:08.82510758Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 776.429103ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:08.993252652Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 1.070639926s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:09.103189086Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 958.996459ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:09.281508584Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 533.746866ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:09.544500309Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 1.216764152s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:09.781017715Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 1.195797164s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:10.076766576Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 1.444991902s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:10.088078761Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 1.090731977s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:10.184269778Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 1.38934626s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:10.250530479Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 887.009564ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:11.007181538Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 1.380616028s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:11.281487027Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 638.695389ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:11.28990653Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 1.084537181s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:11.541555748Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 1.12739055s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:11.648572638Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 869.440068ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:11.901922589Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 785.128304ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:11.935213649Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 559.005554ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:12.635739511Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 1.499648659s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:12.671457983Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 1.082433445s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:12.729811675Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 1.06086531s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:12.7803832Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 994.00733ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:12.788409233Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 1.334434289s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:13.204317405Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 1.031756855s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:13.791660532Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 1.237587389s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:13.820027587Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 1.042226383s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:13.849507391Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 1.207822063s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:14.180587813Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:14.224760997Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46041"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:14.224965282Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"d1bbfbb8-69c3-4b2a-9e8c-d2f75396854a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46041","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":106,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 688.269587ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:14.548867844Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:14.817331686Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"698.133548ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 1.277394316s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:14.833390081Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 943.421725ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:14.9019143Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 1.492990289s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:15.066062471Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 1.264147687s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:15.245712773Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 1.23971728s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:15.396491047Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 714.317056ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:15.854446149Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 975.223311ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:16.120716807Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 1.010153596s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:16.519398355Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 828.029627ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:16.568651057Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 567.263138ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:16.577268835Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 1.20228559s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:16.644973472Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 1.094999572s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:17.10432747Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 1.379953515s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:17.153986403Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 716.37555ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:17.405941909Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 1.318151322s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:17.538010781Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 833.200808ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:17.855249224Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 1.378134738s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:18.131246046Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 1.078335346s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:18.209207843Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 726.97326ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:18.54283322Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 559.452545ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:18.696482146Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 1.126460783s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:18.865266914Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2\") failed: Property not found. retry in 1.151682565s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:19.264762262Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 983.462188ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:19.278735224Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 811.241136ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:19.295439976Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951\") failed: Property not found. retry in 1.36597076s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:19.518015741Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963\") failed: Property not found. retry in 917.971768ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:20.000447311Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a\") failed: Property not found. retry in 922.914769ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:20.116527369Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121\") failed: Property not found. retry in 1.057641972s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:20.271109229Z","hostname":"sock","pid":1590,"zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:21.352050948Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_5a440a8c-c4c1-44e2-a8b9-db083615d951"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:21.352118779Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_c1418514-c5cf-4740-8456-b388e685986a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:21.35212869Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_c4807d78-ab52-48f3-8c04-b82c898865f2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:21.35213825Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_8625b445-de69-4e1e-9b9f-bf8bc8e2a963"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:21.35214612Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_0d90006a-1e7a-41dc-97c1-6c426626b121"} {"msg":"Writing ledger to /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:21.35215952Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:21.352590371Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:21.35294014Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"106a8a5b-f16c-4814-85be-6249072c6a84","remote_addr":"[fd00:1122:3344:101::1]:63723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":23182008,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:21.404410259Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:41007"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:21.438520159Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3437"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:39.318826195Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3455"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.394119574Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.396410002Z","hostname":"sock","pid":1590,"uri":"/cockroachdb","method":"POST","req_id":"712b50a7-39cc-44e3-a368-6bcba49b2849","remote_addr":"[fd00:1122:3344:101::1]:41007","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":24991883,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.400247499Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58597"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.402413803Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"b282bee4-dc63-4985-8688-72bc5fe65901","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58597","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1148,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:46.414231001Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.347567204s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.453375049Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.454412505Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:64461"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.456790465Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.4569766Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.481920759Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.48197117Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.502196891Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.502256772Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.521948759Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.52199212Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.541098772Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.541152683Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.559373263Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.559423744Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.578817064Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.578865315Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.597178447Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.597236988Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.61555942Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.615617942Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.850431185Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:46.850489897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:47.05673664Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:47.056791781Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:47.270199495Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:47.270252976Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:47.46267309Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:47.462726012Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:47.659368852Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:47.659422884Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:47.822180609Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:47.82222213Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.006504879Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.006559901Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.178042597Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.178099958Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.33869314Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.338745471Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.513380276Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.513447368Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.72877513Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.728833182Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.94044885Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.940509982Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.941565208Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:48.942272356Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106577771Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106627702Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106633673Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106693694Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106705964Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106716675Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106725985Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106735975Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106750096Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106771396Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106789546Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106800457Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106818567Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106831498Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106846008Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106864798Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106876349Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106895639Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.106913589Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.107085134Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"b32804a4-cdff-4b5d-a56d-9ef39e890693","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized)]","latency_us":2652136,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.107498224Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"118.322913ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b32804a4-cdff-4b5d-a56d-9ef39e890693\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b32804a4-cdff-4b5d-a56d-9ef39e890693\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.227520332Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.228345023Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.228351923Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.250067321Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.250111222Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.269236164Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.269278766Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.28847241Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.288512211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.306386572Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.306423363Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.325442002Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.325476863Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.344628326Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.344667938Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.364554509Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.36460627Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.383667361Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.383708362Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.403070251Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.403107592Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.423037935Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.423075925Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.441300065Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.441339386Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.459236138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.459286609Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.479268323Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.479310824Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.49815597Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.498194761Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.517019906Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.517058507Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.535716137Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.535765138Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.554097141Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.554136262Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.573570962Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.573614173Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.592226483Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.592273574Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.612077104Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.612119575Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.61232468Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.612491644Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.70508352Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705145461Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705155492Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705162212Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705170412Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705177082Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705185653Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705193833Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705200883Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705209233Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705217453Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705223963Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705232254Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705237244Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705245334Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705252044Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705277495Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705284595Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705291205Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.705414508Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"6f69d23e-9e4d-4ca4-a8e6-4e3780ce8824","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized)]","latency_us":477254,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:49.705596873Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"178.254026ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6f69d23e-9e4d-4ca4-a8e6-4e3780ce8824\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6f69d23e-9e4d-4ca4-a8e6-4e3780ce8824\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.884986488Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.885332246Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.885341057Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.906432769Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.90647416Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.927265534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.927326136Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.9465209Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.946563131Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.965506299Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.96554414Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.984503128Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:49.984546019Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.003737793Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.003783295Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.027112093Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.027163284Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.048603475Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.048647606Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.067767039Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.06781032Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.088682947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.088721188Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.107159462Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.107201274Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.126546622Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.126595773Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.145732026Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.145775457Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.165960426Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.166000547Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.186090034Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.186131335Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.205407101Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.205455922Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.225026316Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.225073427Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.243943203Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.243987274Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.263369403Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.263413204Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.282582088Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.28263878Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.282971368Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.283107221Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381320099Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.38137151Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.38138Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381386051Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381391871Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381397391Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381402971Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381409651Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381414801Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381419832Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381426642Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381432432Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381437942Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381441902Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381445712Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381451002Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381460953Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381464933Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381470013Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.381546465Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"1c32fda0-7263-4f38-b1bb-24af01957291","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized)]","latency_us":496393,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:50.381735789Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"157.522486ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1c32fda0-7263-4f38-b1bb-24af01957291\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1c32fda0-7263-4f38-b1bb-24af01957291\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.540173416Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.540477804Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.540492554Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.562979102Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.563028703Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.583380266Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.583438158Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.603261768Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.603311129Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.621092798Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.621128629Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.638806065Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.638838515Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.656533082Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.656568363Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.674094675Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.674136806Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.693123785Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.693157226Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.710920834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.710956625Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.729260127Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.729303258Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.746947733Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.746982234Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.766744362Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.766794923Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.785601008Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.785637869Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.805487499Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.805525251Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.823298339Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.823331819Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.840869042Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.840919843Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.860992699Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.861030951Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.880558613Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.880600374Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.898835914Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.898872755Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.91811534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.918172712Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.918347486Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:50.918469629Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013879826Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013925897Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013934388Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013939678Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013945428Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013952408Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013958298Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013962529Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013968109Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013974049Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013980909Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013986499Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013991979Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.013997639Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.01401071Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.01402545Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.014048371Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.014054901Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.014060441Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.014199664Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"6426d399-f51b-4cbc-9685-06d27e8cba83","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized)]","latency_us":473900,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.014389369Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"197.834892ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6426d399-f51b-4cbc-9685-06d27e8cba83\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6426d399-f51b-4cbc-9685-06d27e8cba83\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.212854296Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.213239896Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.213258316Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.236716438Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.236759579Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.25661334Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.256671071Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.276466211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.276509762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.297181463Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.297230415Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.316807238Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.316846879Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.336019233Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.336056494Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.355083984Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.355121485Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.373807756Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.373844197Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.392219091Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.392254902Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.410466021Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.410502132Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.429526332Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.429567763Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.448597743Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.448728026Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.467769757Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.467810578Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.486947781Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.486982582Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.505510719Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.50554805Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.524330634Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.524375475Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.544274827Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.544322548Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.562969459Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.56300798Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.58284513Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.582963703Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.60107522Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.601133121Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.60147209Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.601652644Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696079366Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696135788Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696144838Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696153338Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696159488Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696167379Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696173519Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696179789Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696187879Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696195629Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.69620207Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.69621026Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.69621634Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.69622452Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.6962325Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.69623753Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696250071Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696255141Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696259661Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.696340633Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"978a80a4-d1ae-4b05-9511-a7fc930c48d5","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized)]","latency_us":483320,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:51.696534698Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"211.132175ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"978a80a4-d1ae-4b05-9511-a7fc930c48d5\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"978a80a4-d1ae-4b05-9511-a7fc930c48d5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.908942626Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.909278185Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.909289925Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.93088869Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.930936501Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.951089379Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.951147491Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.970769836Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.970813337Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.989239382Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:51.989277373Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.006528918Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.006570729Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.024295366Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.024333837Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.044137117Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.044175448Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.061914695Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.061953956Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.081099669Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.0811372Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.098624801Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.098661452Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.116543413Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.116581734Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.135279546Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.135324507Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.153487136Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.153524597Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.171549531Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.171582942Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.189115384Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.189154865Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.208142204Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.208191336Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.225681757Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.225716828Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.245094807Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.245134208Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.263889001Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.263929262Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.282016018Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.282073869Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.282242794Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.282363597Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373316721Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373356892Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373362082Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373366323Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373370172Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373374723Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373378663Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373486706Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373494076Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373505316Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373512656Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373518276Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373524047Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373539657Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373545697Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373567888Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373574058Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373580268Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373591158Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.373729572Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"f81ac550-565b-4e8d-8ac1-ebcc647d082d","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized)]","latency_us":464653,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:52.373923117Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"224.196849ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f81ac550-565b-4e8d-8ac1-ebcc647d082d\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f81ac550-565b-4e8d-8ac1-ebcc647d082d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.599335463Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.599641201Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.599648571Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.62418371Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.624229791Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.645692433Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.645747074Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.666469277Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.666517068Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.68561111Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.685656001Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.704886026Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.704928367Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.72287448Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.722919391Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.741839078Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.741897949Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.759800781Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.759846522Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.778935784Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.778972545Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.798494537Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.798538589Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.819050126Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.819092737Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.838405224Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.838449445Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.85729922Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.857340912Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.876528505Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.876584947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.895405582Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.895453333Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.914889413Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.914936614Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.93379805Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.933980525Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.952706897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.952749479Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.972172669Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.972211199Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.992961753Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.993022054Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.993223569Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:52.993348293Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089345114Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089400776Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089410306Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089415726Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089423676Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089430376Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089436777Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089441307Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089447187Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089451617Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089456117Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089460597Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089465128Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089471117Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089477098Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089489838Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089496298Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089502358Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089506818Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.08956049Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"75d002fb-b9b0-45c1-812e-7f505f8abf04","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized)]","latency_us":490085,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.089768005Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"156.24954ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"75d002fb-b9b0-45c1-812e-7f505f8abf04\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"75d002fb-b9b0-45c1-812e-7f505f8abf04\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.247183386Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.247514975Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.247526025Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.26991463Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.269964601Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.288869998Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.288920109Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.308333329Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.30838319Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.327661906Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.327703807Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.346718167Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.346760058Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.364646719Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.36468802Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.38289099Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.382937721Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.400677328Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.40072294Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.418286533Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.418327684Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.43641293Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.436460361Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.454218669Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.45426622Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.471722741Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.471763322Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.489677624Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.489726165Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.508201661Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.508243852Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.525927268Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.52597744Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.542912277Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.542974678Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.561429574Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.561482775Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.578810592Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.578851383Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.5973435Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.597385371Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.614702328Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.614759309Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.614940654Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.615056197Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704731219Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.70478531Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.70479406Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704800031Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704805691Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704811981Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704819671Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704823921Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704829512Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704833601Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704839172Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704845852Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704850872Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704856442Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704863222Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704884613Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704889203Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704894523Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.704898423Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:53.704982855Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"77d54a06-164c-4046-b508-1120d5d87412","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized)]","latency_us":457638,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:53.705192981Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"479.62313ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"77d54a06-164c-4046-b508-1120d5d87412\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"77d54a06-164c-4046-b508-1120d5d87412\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.186731008Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.187065957Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.187091067Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.210012265Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.210059337Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.231032706Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.231088327Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.251280817Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.251350478Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.270877561Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.270922292Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.290705051Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.290761273Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.308758777Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.308812508Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.328863414Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.328926395Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.347614387Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.347655108Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.366886743Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.366929094Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.38580263Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.385846461Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.404901852Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.404960834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.423175083Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.423220124Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.441808283Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.441849344Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.4603006Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.460346021Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.478613882Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.478659863Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.496472892Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.496540634Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.514819485Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.514878947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.533357643Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.533400244Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.552845564Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.552894055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.571767752Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.571837853Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.572071379Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.572205283Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663538447Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663588878Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663594008Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663599588Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663605199Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663610509Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663617449Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663622959Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663628509Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663633859Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663639039Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663645849Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.66364996Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.66365707Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.66366239Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.66367296Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.66367999Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663683941Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.66368779Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:54.663761592Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"97605f89-b971-4f67-8bfe-9ebc80e5aec3","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized)]","latency_us":476918,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:54.663954907Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"547.782561ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"97605f89-b971-4f67-8bfe-9ebc80e5aec3\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"97605f89-b971-4f67-8bfe-9ebc80e5aec3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.21348122Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.213836379Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.213845759Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.235857844Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.235918196Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.256955507Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.257019068Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.276960572Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.277009613Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.295770076Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.295813727Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.314462307Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.314505388Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.333867577Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.333916958Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.353228876Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.353279287Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.372896542Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.372938133Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.392574838Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.392617159Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.412233124Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.412280725Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.431943421Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.431998733Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.450627322Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.450668204Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.468795471Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.468846282Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.486556179Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.48660162Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.505175079Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.50521588Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.523766348Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.523823139Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.542937911Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.542996133Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.560989947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.561035078Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.579952295Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.580007057Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.598859192Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.598913554Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.599089728Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.599204931Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691244423Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691303954Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691312025Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691320365Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691328935Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691335555Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691343355Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691349515Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691355866Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691363936Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691368916Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691373536Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691379836Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691384636Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691389436Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691412867Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691419587Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691423997Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691428307Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:55.691475098Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"790484b5-01da-4c7e-a7fe-6f447ba9f526","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized)]","latency_us":477836,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:55.691677294Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"408.082895ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"790484b5-01da-4c7e-a7fe-6f447ba9f526\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"790484b5-01da-4c7e-a7fe-6f447ba9f526\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.101149753Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.101490072Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.101497492Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.124705877Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.124755248Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.144727132Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.144781974Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.164715927Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.164767358Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.185406458Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.18544882Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.203189287Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.203236698Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.22153694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.221593911Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.240232822Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.240290993Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.258365799Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.25841277Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.276823885Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.276870556Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.294553752Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.294605733Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.311851919Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.31190791Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.328101258Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.32815213Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.344373699Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.34441806Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.362041565Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.362087786Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.382741167Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.382794448Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.401265214Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.401314265Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.418425407Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.418453648Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.435875247Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.435916008Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.455547153Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.455584234Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.474085891Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.474141013Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.474340988Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.474486051Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.56402953Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564093312Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564107502Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564115032Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564120503Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564128743Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564134053Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564139313Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564143473Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564148683Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564152333Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564157304Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564162484Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564167664Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564171354Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564184174Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564187934Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564193064Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.564197194Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:56.564211335Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"c511e543-7ab9-4564-8dc5-29fcbad2c8fa","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized)]","latency_us":462906,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:56.56440971Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"653.107775ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c511e543-7ab9-4564-8dc5-29fcbad2c8fa\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c511e543-7ab9-4564-8dc5-29fcbad2c8fa\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.218003428Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.218303696Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.218310936Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.240548207Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.240603478Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.260571102Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.260628923Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.280764802Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.280820813Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.299662178Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.299715649Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.320410551Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.320456533Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.339918784Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.339966375Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.357829135Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.357886147Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.376662941Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.376706932Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.396429609Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.396487201Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.415840569Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.41588355Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.435820803Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.435870694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.455597552Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.455719285Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.475301039Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.47534198Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.494911914Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.494962215Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.514553489Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.514603181Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.534887762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.534946784Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.554744143Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.554791434Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.573551038Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.573593039Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.592571597Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.592616759Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.610628993Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.610685054Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.61091946Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.611067604Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703196218Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.70326825Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.70327986Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703286121Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703294501Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703301201Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703308111Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703316691Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703323282Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703329772Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703338012Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703344602Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703352592Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703359382Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703366303Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703392483Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703399653Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703414784Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703437454Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:57.703613079Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"fd28f1a6-9e30-4e73-a998-10559cfae3ac","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized)]","latency_us":485460,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:57.703847005Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"451.511925ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fd28f1a6-9e30-4e73-a998-10559cfae3ac\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fd28f1a6-9e30-4e73-a998-10559cfae3ac\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.156333039Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.156659718Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.156669468Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.179644958Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.179694359Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.202084644Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.202127354Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.222711694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.222762925Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.241277732Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.241317863Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.260129418Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.260174599Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.279725942Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.279770813Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.300389273Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.300443495Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.320167103Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.320215244Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.339520561Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.339568372Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.358235153Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.358277544Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.377499769Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.37755131Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.396881228Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.396919199Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.415394585Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.415438716Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.43422277Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.434276141Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.453340262Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.453394853Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.47229574Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.472344422Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.491112065Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.491163126Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.50995058Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.510000852Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.528777105Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.528821826Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.54720359Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.547276332Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.547479247Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.54762049Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640163805Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640247007Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640258558Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640268768Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640278178Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640287418Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640294748Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640303629Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640309189Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640316899Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640322359Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640327539Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640332909Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640340289Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.64034754Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.64036749Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.64037715Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640382651Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640389631Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:58.640403051Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"27808593-c3ff-4cc9-8a10-4cff5e7d3a57","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized)]","latency_us":483935,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:20:58.640617087Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.329785644s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"27808593-c3ff-4cc9-8a10-4cff5e7d3a57\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"27808593-c3ff-4cc9-8a10-4cff5e7d3a57\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:59.971391627Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:59.971915131Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:59.971928181Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:59.993497625Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:20:59.993607568Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.01233818Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.012382272Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.030051627Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.030103669Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.047568349Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.04760796Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.06582822Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.065871901Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.083508376Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.083552907Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.101241883Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.101292175Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.119411542Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.119466463Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.136937744Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.136989245Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.155915653Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.155955964Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.174386628Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.17443469Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.19188485Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.191930921Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.209748051Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.209799352Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.227251472Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.227295653Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.244567819Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.24461873Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.262279006Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.262329157Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.280038644Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.280091535Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.298569401Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.298605792Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.317027947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.317066958Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.33499334Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.335069172Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.335259287Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.33537155Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.42574836Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425801381Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425809591Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425813942Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425820902Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425827062Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425834422Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425839892Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425846092Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425853152Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425858803Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425868063Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425881803Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425892103Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425915924Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425934105Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425952805Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425969315Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.425979346Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:00.426125449Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"e8e8886d-5489-401f-90f5-072b987c310b","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized)]","latency_us":454529,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:00.426324484Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"615.952414ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e8e8886d-5489-401f-90f5-072b987c310b\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:20:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e8e8886d-5489-401f-90f5-072b987c310b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.042841307Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.043193766Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.043201346Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.06592815Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.06597294Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.086472098Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.086514099Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.108055602Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.108101653Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.127319558Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.127366339Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.145820515Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.145867786Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.164968478Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.165015909Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.183614308Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.18367765Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.201751146Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.201801197Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.221295689Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.22133904Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.239781145Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.239824366Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.258079447Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.258145649Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.275690391Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.275740472Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.293723026Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.293774367Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.311873754Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.311928435Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.331645923Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.331699034Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.350870688Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.350917089Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.369617451Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.369684642Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.38824029Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.388291812Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.408054401Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.408104622Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.427810519Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.427876901Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.428083556Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.428233239Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520649951Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520707292Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520718713Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520725633Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520734503Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520741113Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520749003Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520755333Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520763064Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520770964Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520777804Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520785504Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520791834Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520796485Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520810015Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520815015Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520819705Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520827515Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.520832226Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:01.520905587Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"270a8c5d-fb2a-402b-8302-c4b42905a015","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized)]","latency_us":477896,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:01.521084742Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.102953746s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"270a8c5d-fb2a-402b-8302-c4b42905a015\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"270a8c5d-fb2a-402b-8302-c4b42905a015\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.625699318Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.626026006Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.626035256Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.649355075Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.649402936Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.668752054Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.668804735Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.689389455Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.689434976Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.707000559Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.7070456Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.725613788Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.7256568Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.743325055Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.743376247Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.761292159Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.76133737Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.780525314Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.780572945Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.801044731Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.801101323Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:02.808673334Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"2.116098823s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.820109612Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.820154593Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.838626439Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.838675411Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.856986252Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.857031834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.87550212Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.875557491Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.895083123Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.895132285Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.913600611Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.913654072Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.932030575Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.932075837Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.950689446Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.950736067Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.969583313Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.969624614Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.98927897Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:02.989333131Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:03.008138506Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:03.008197227Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:03.008424653Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:03.008559896Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.107315917Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.107377419Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.107385769Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.107391799Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.10739714Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.10740427Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.10740984Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.10741524Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.10742224Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.10742784Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.10743336Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.107440251Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.107463001Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.107470161Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.107476912Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.107482202Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.107489112Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.107495152Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.107501662Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:03.107629716Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"a1710c9a-4e39-4b34-8931-3be684c6ffa7","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized)]","latency_us":481789,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:03.10782883Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"1.849619958s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a1710c9a-4e39-4b34-8931-3be684c6ffa7\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a1710c9a-4e39-4b34-8931-3be684c6ffa7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:04.957850741Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:04.958144188Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:04.958156759Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:04.980372809Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:04.980441231Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:04.999744718Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:04.999790289Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.021524555Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.021577431Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.041288436Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.04133157Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.060785359Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.06082393Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.079569855Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.07960835Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.098381917Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.098431793Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.116296093Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.116347808Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.134411835Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.13444942Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.151999756Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.152052539Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.17059018Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.170628365Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.188550525Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.188598038Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.20627145Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.206316057Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.223706244Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.223739217Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.241748309Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.241795589Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.259044134Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.259090382Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.278354108Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.278399598Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.296573019Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.296613501Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.315143446Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.315188092Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.334431782Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.33448854Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.334719152Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.33487901Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425474157Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425518784Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425526687Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425531946Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.42553849Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425544809Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425550022Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.42555501Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425559829Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425565876Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425570929Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.42557586Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425590945Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425596355Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425599795Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425603226Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425608017Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425611523Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.42561487Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:05.425680553Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"f1172baf-80b3-4766-be89-a97a8be7849c","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized)]","latency_us":467286,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:05.425878249Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"2.182269945s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f1172baf-80b3-4766-be89-a97a8be7849c\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f1172baf-80b3-4766-be89-a97a8be7849c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.612458624Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.612825402Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.612834112Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.635707461Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.635755499Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.657082024Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.657115906Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.678016477Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.678055844Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.696578386Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.696623837Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.715636684Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.71568088Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.734107431Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.734153126Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.752667297Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.752700421Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.770236517Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.770269612Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.788553791Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.788587007Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.807414554Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.807454249Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.825462324Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.825498569Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.84384567Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.843883696Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.862371817Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.862528712Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.880816677Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.880854882Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.898949216Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.899002486Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.916975575Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.917008642Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.93551089Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.935543675Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.956194923Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.956239661Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.974445397Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.974479289Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.995054143Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.995107347Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.995404013Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:07.995526997Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.09249732Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092555213Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092563285Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092570119Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092574835Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092579888Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092586489Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092591476Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092596642Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092603598Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.09260897Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092614211Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092636665Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092642477Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092648008Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092653493Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092660327Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092665372Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092670416Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:08.092765442Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"baea806a-21f1-48f4-b406-4da27c24d312","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized)]","latency_us":479637,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:08.092970761Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.063732588s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"baea806a-21f1-48f4-b406-4da27c24d312\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"baea806a-21f1-48f4-b406-4da27c24d312\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.16079471Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.161121774Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.161130043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.183499384Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.183551772Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.202922695Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.202973583Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.220612524Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.220722621Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.237320842Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.237356487Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.254561752Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.254603172Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.271222311Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.27126538Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.288672007Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.288709292Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.307951872Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.307984404Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.327122929Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.327152293Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.345274766Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.345332076Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.362665893Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.362702803Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.380317662Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.380362382Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.397524815Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.397565607Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.415400384Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.415435007Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.433153727Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.433195278Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.451628588Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.451666051Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.468618685Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.468652651Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.486826122Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.48686248Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.506703192Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.506741611Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.525476776Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.525532249Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.525723063Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.525828385Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619267092Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619318384Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619334913Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619350044Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619357938Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.61936542Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.61937008Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619375686Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619385155Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619390048Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619400792Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619407327Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619425252Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619436812Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619441322Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619447303Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619451681Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619455872Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.619466841Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:11.619471444Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"f9eba524-5531-418c-99be-9d24b8b794e4","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized)]","latency_us":458052,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:11.62016395Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.023973262s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f9eba524-5531-418c-99be-9d24b8b794e4\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f9eba524-5531-418c-99be-9d24b8b794e4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.648518755Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.649625097Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.649634509Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.670928278Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.670984492Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.691547507Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.691593146Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.709717072Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.709774186Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.730248166Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.730291732Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.749268787Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.749328648Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.768082935Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.768145129Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.786113764Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.786174609Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.804616065Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.804672391Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.82315583Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.823209887Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.841677404Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.841735522Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.860080707Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.860128464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.878340341Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.878404261Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.896372029Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.896421718Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.914870183Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.914922385Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.934933686Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.935004244Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.954811849Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.954868654Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.97348424Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.975935466Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.992519049Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:14.992577071Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:15.012156919Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:15.012199998Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:15.030132943Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:15.030195392Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:15.03042051Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:15.030565817Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.124786047Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.124855265Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.124864236Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.124935627Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.124951424Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.124957593Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.12497068Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.124976924Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.124986337Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.125002031Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.125008144Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.125020895Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.125039082Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.125045532Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.125050013Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.125054514Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.125060467Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.125064789Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.125069467Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:15.125074014Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"35ce1be6-408a-42fe-97e8-b99f9163b6ca","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized)]","latency_us":475868,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:15.12526976Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"4.582959181s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"35ce1be6-408a-42fe-97e8-b99f9163b6ca\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"35ce1be6-408a-42fe-97e8-b99f9163b6ca\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:16.451512365Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57911"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:16.451560385Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"6c59c475-4200-405e-9b37-8fda2a53fb42","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57911","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.713973802Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.714470615Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.714486965Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.73736357Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.737412686Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.757743837Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.757795044Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.7770052Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.777046816Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.795982855Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.796029478Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.814899739Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.815018937Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.832906934Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.832951617Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.851341327Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.851381472Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.870841222Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.870883954Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.889826736Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.889904017Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.908813925Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.908857501Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.927376297Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.927414932Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.946353407Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.946399223Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.965637037Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.965686557Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:19.981083709Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"2.791726042s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.985721528Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:19.98573515Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.004800266Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.004850938Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.024766572Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.024807673Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.043855583Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.043901512Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.064047111Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.064111885Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.083530634Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.08357796Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.102708264Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.102765293Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.102968221Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.103089189Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197506131Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197576943Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197585043Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197590602Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197603578Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197609672Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197626369Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197632406Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197639334Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197662285Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197669898Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197677427Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197684186Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197689867Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197697133Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197702149Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197708984Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197712836Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197716709Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:20.197783648Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"06c6084a-bd0b-48aa-b65c-a2a4f638ddd9","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized)]","latency_us":482956,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:20.197982506Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"4.120906387s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"06c6084a-bd0b-48aa-b65c-a2a4f638ddd9\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"06c6084a-bd0b-48aa-b65c-a2a4f638ddd9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.324259126Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.324712521Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.324729134Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.346430939Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.346498254Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.367177171Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.367228088Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.385607109Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.385666305Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.405673345Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.408016864Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.424673723Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.424718293Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.443006715Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.443078653Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.461777199Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.461824544Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.480443518Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.480498512Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.499421382Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.499479396Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.520475243Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.520523638Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.53984764Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.539904501Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.557731828Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.557788388Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.57603078Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.576174579Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.594731393Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.594782928Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.612355343Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.612410722Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.631248991Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.631303546Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.649510298Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.649561365Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.668262985Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.668330637Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.685945825Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.68601418Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.704014016Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.704097307Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.704390548Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.704519738Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794198396Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794254694Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794263788Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.79428101Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794295336Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794303689Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794393341Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794411623Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794418289Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794424701Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794429398Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794433937Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794438193Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794444643Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794452433Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794464491Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.7944753Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794479275Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794485266Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:24.794521754Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"a8fb98f4-c8ea-441f-8b52-63278c8a5ae3","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized)]","latency_us":469553,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:24.794729537Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"6.856894864s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a8fb98f4-c8ea-441f-8b52-63278c8a5ae3\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a8fb98f4-c8ea-441f-8b52-63278c8a5ae3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.66032177Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.660631565Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.660640753Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.681830324Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.681902241Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.701497636Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.701557329Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.719872349Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.719930793Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.737067403Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.73713152Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.754364102Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.754409984Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.771195045Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.771241762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.788331819Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.788387311Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.806072103Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.806133802Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.82286165Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.822977222Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.839568662Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.839626517Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.857885606Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.85800967Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.87525655Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.875315023Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.890930521Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.891049303Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.908395731Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.908450895Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.925222009Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.925275775Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.9473434Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.94739878Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.965937056Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.965988207Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.982452297Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:31.982516743Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:32.000377256Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:32.000438767Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:32.018077242Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:32.018200841Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:32.018400167Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:32.018530351Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.1080144Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108084676Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108095355Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108109512Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.10811554Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108123724Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108145231Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108154625Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108164394Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108183276Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108199523Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108208579Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.10830586Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108315329Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108320917Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108334033Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108340145Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108346033Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.10835299Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:32.108422741Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"aff913eb-750b-4389-be37-f60a84feb280","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized)]","latency_us":447450,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:32.108659963Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"3.669695723s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aff913eb-750b-4389-be37-f60a84feb280\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aff913eb-750b-4389-be37-f60a84feb280\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.783523365Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.783803553Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.7838132Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.805797964Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.805852883Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.826484666Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.826551933Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.845523264Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.845576281Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.864146903Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.864205611Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.88323359Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.883361205Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.901008204Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.901058643Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.91850704Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.91855822Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.936206036Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.936259953Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.954757916Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.954805299Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.972816238Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.972874749Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.991482769Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:35.991531399Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.009921278Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.009966644Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.029586633Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.029687594Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.049037989Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.049097034Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.06876758Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.068810622Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.087473095Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.087590763Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.106273272Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.106316454Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.124749282Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.124809124Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.143781305Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.143827694Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.162080594Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.162158577Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.162366117Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.162490141Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252458882Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252528924Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252541862Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252560791Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252572013Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252584931Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252613554Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252622855Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252635343Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252642393Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252656286Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.25266224Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252667866Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252676997Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252682856Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252757754Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252767636Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252774423Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252781698Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:36.252787501Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"bd85d4e1-e66f-450a-811a-0a26e25e9cf2","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized)]","latency_us":468579,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:36.252987579Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"5.61427537s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bd85d4e1-e66f-450a-811a-0a26e25e9cf2\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bd85d4e1-e66f-450a-811a-0a26e25e9cf2\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:37.829942494Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"8.858616076s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.871848797Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.872212306Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.872223116Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.896290753Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.896379546Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.919432347Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.919490638Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.940092598Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.94015742Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.95880046Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.958861512Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.979268087Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.979341359Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.999845766Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:41.999898657Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.019332387Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.019395709Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.037051054Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.037123896Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.056015423Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.056070044Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.075446383Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.075503974Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.091924529Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.09198124Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.111427561Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.111482662Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.132192544Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.132251276Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.15301821Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.153069791Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.174687287Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.174743868Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.195410869Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.195469911Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.215413454Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.215478276Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.236815044Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.236871455Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.257595408Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.25765572Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.278016313Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.278082625Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.278330901Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.278495925Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376032186Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376110288Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376122318Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376138579Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376146299Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376152519Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376158589Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376165889Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.37617337Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.37617967Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.37618863Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.37621648Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376228711Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376235411Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376243181Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376249901Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376254112Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376260042Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.376264262Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:42.376354694Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"c5263ede-fe8f-45de-adcc-d1baedc59e5b","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized)]","latency_us":504323,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:42.37659545Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"8.774505722s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c5263ede-fe8f-45de-adcc-d1baedc59e5b\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c5263ede-fe8f-45de-adcc-d1baedc59e5b\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:46.510433185Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39730"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:46.510510497Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"1e66513c-8dcf-4a6b-9e73-f2ab32bc9b8d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:39730","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.152092249Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.152462989Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.152472499Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.175309865Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.175373177Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.195262379Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.195342921Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.213514399Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.21357086Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.231440841Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.231516923Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.248754868Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.24881134Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.265974693Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.266027224Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.284232023Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.284281724Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.301260703Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.301322554Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.318557339Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.318623901Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.336475051Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.336525762Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.353796518Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.35386583Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.371392072Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.371446583Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.389669423Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.389723654Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.408539919Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.408595451Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.428275677Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.428331428Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.447916403Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.45021216Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.466021319Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.466085371Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.48429923Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.484364632Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.502845838Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.50290425Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.520472513Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.520539384Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.5207596Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.520910474Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.61075398Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610845173Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610854633Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610863963Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610871013Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610879614Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610890014Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610898704Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610906774Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610912974Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610920404Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610932735Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610940115Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610946695Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610954105Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610968846Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610976546Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610982416Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.610989846Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:51.611094129Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"6b5f977c-5e78-4fa3-8a86-3051ae65dfcb","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized)]","latency_us":458858,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:21:51.611323845Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"8.072652508s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6b5f977c-5e78-4fa3-8a86-3051ae65dfcb\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6b5f977c-5e78-4fa3-8a86-3051ae65dfcb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.685059691Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.685369289Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.685377719Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.707463046Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.707519128Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.727494482Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.727552353Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.747220089Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.747276161Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.764933956Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.764985218Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.782754546Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.782808557Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.800935144Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.800988696Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.818371784Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.818431286Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.836251805Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.836314837Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.854522616Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.854583117Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.872561391Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.872607942Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.890076713Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.890127574Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.908498658Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.908563179Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.926231035Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.926280096Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.944517576Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.944570808Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.962245133Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.962305575Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.980773211Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.980839172Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.999094843Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:21:59.999147284Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:00.01798886Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:00.018041941Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:00.036920187Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:00.036970739Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:00.056331597Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:00.056396138Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:00.056599974Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:00.056768478Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148598535Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148666676Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148683387Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148704717Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148718218Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148724308Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148729608Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148736998Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148745688Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148752959Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148761979Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148768899Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148777769Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148784819Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.14880255Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.14882691Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.14883599Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148845391Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.148851121Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:00.148961374Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"29676127-2730-4cf9-8ba0-af285886be6b","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\", TimeNotSynchronized), (\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\", TimeNotSynchronized), (\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\", TimeNotSynchronized), (\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\", TimeNotSynchronized), (\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\", TimeNotSynchronized), (\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\", TimeNotSynchronized), (\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\", TimeNotSynchronized), (\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\", TimeNotSynchronized), (\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\", TimeNotSynchronized), (\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\", TimeNotSynchronized), (\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\", TimeNotSynchronized), (\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\", TimeNotSynchronized), (\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\", TimeNotSynchronized), (\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\", TimeNotSynchronized), (\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\", TimeNotSynchronized), (\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\", TimeNotSynchronized), (\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\", TimeNotSynchronized), (\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\", TimeNotSynchronized), (\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\", TimeNotSynchronized)]","latency_us":463761,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:00.149290612Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:467","retry_after":"14.999774944s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"29676127-2730-4cf9-8ba0-af285886be6b\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:21:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"29676127-2730-4cf9-8ba0-af285886be6b\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:01.728349307Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"21.909129837s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fc04604b-21ab-4a40-a700-a181ed4f060d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: ca69eda0-d0a3-43c2-973e-0e528820dde8, kind: Service { id: fc04604b-21ab-4a40-a700-a181ed4f060d }, name: Name(\"ntp-fc04604b-21ab-4a40-a700-a181ed4f060d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 149, 130])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: c4807d78-ab52-48f3-8c04-b82c898865f2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d90006a-1e7a-41dc-97c1-6c426626b121, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: c1418514-c5cf-4740-8456-b388e685986a, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f47e1510-c515-4500-92e8-33a6349000e8, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: ba162515-e11a-42da-aa7b-aaf2fab53588, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 420a9926-428f-4a96-ac2a-14aba90f476b, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 2584e6b7-8bd3-4591-b4dd-c3d456b7896a, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7409f646-a2ae-4503-8528-69a75169324e, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: aa9efd65-76e0-40d3-b044-475ef541089b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 66cf0e2c-6a1d-491a-b5ab-d43fdc957b77, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 536bddb2-c38c-4e67-8b5d-9f1f1fb986e0, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 912c51fb-2723-4253-a072-17759e0be466, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External } } } }, OmicronZoneConfig { id: ed844809-41fb-4820-b590-15e558d9334b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External } } } }, OmicronZoneConfig { id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External } } } }, OmicronZoneConfig { id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External } } } }, OmicronZoneConfig { id: 70c53f9a-7915-490e-bd40-caf245b59e8f, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External } } } }, OmicronZoneConfig { id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External } } } }, OmicronZoneConfig { id: 055c12a1-5182-45fc-8015-ccd05d39636c, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8b417-316d-4066-8027-98c8dac8136b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8c14321b-a1bb-4420-abee-713b270542b8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.149385441Z","hostname":"sock","pid":1590,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:429"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.149673378Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.149681918Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fc6accb5-93bc-4f13-b90c-1fe0dce9aa08, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.172800082Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.175452928Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 070282d1-0314-4f24-9b9b-f3fe2fd417c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.192314174Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.192372445Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 73cb0d1c-76b9-46e7-a3c4-339faca846c1, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.212126703Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.212189485Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5a440a8c-c4c1-44e2-a8b9-db083615d951, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.230558099Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.23061752Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c4807d78-ab52-48f3-8c04-b82c898865f2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.248102461Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.248158652Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0d90006a-1e7a-41dc-97c1-6c426626b121, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.265955552Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.266028943Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8625b445-de69-4e1e-9b9f-bf8bc8e2a963, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.283212627Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.283261238Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1418514-c5cf-4740-8456-b388e685986a, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.301344534Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.301402626Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b369e86a-ba32-4e8e-81d4-099acec9e7b1, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.319568174Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.319632426Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bb8e6fc4-c8a5-446a-bd73-00970284aef2, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.337972268Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.33804617Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7409f646-a2ae-4503-8528-69a75169324e, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.356490405Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.356562837Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 912c51fb-2723-4253-a072-17759e0be466, dataset_name: DatasetName { pool_name: ZpoolName { id: 9a1451e3-b124-44f6-8078-fbc416821fc9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.375568837Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.375629538Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ed844809-41fb-4820-b590-15e558d9334b, dataset_name: DatasetName { pool_name: ZpoolName { id: 52a73ea1-1580-48fa-9393-5bb705994a44 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.393468748Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.39353732Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 99d63837-89b8-441d-8f65-e85c20ba3ad6, dataset_name: DatasetName { pool_name: ZpoolName { id: 03d465fc-bc73-4466-af31-1a5670be772e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.411866402Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.411927254Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb, dataset_name: DatasetName { pool_name: ZpoolName { id: 145ad0b0-5e5c-4755-ba6c-da149e549070 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.430912343Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.430962834Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70c53f9a-7915-490e-bd40-caf245b59e8f, dataset_name: DatasetName { pool_name: ZpoolName { id: b110b23f-2145-46b8-8de0-dc91bc41dd55 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.449131843Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.449184584Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 83e7a4a6-713d-420b-a2b9-eebccafbd808, dataset_name: DatasetName { pool_name: ZpoolName { id: d3fbf4c0-71ec-480f-942f-6fd5fd537b7c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.467755463Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.467813164Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 055c12a1-5182-45fc-8015-ccd05d39636c, dataset_name: DatasetName { pool_name: ZpoolName { id: c8e411fe-43b9-47e1-88c5-1303772b3e4e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.485999013Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.486057504Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6fa8b417-316d-4066-8027-98c8dac8136b, dataset_name: DatasetName { pool_name: ZpoolName { id: e10de419-fa2f-479e-b380-a88997c87dc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.504699464Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.504755696Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8c14321b-a1bb-4420-abee-713b270542b8, dataset_name: DatasetName { pool_name: ZpoolName { id: bb74811e-acf4-486b-913b-bc09ce8f6984 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.523216852Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.523327234Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.523600371Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.523767425Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.649096567Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.649459947Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.649501377Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.649520828Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.649555739Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.69519157Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.695547929Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.69558655Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.695620301Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.695662132Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.860210493Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.860617593Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.897524544Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.897782861Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.897839742Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.897880353Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.899279328Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.912548043Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.912578884Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.913148588Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.918303198Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.922931725Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.927400508Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.934008105Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.939135834Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.943891214Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:15.948439878Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.06462413Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.069501873Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.074222222Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.078558721Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.083314051Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.087204119Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.091509438Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.096404541Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.10189766Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.106948117Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.111621875Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.117048272Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.12448094Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.130747728Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.137243232Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.144479404Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.149836549Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.15540972Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.162252983Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.168171932Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.218734417Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.266310898Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.300993033Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.334686003Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.368077785Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.369657475Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.369952572Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.428019837Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 222, 159]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.428088239Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.431622898Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.431632218Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.485758834Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 150, 90]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.485825966Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.489551629Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.48957555Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.542396023Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 158, 198]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.542443814Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.544473035Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44974"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.544653379Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.54466153Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.54468234Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"32a78ec1-430f-4a72-affb-1b0404bdfb0f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44974","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":104,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.595777559Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 251, 235]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.595824281Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.597592565Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.597609575Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.645110554Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 156, 33]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.645167395Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.645661228Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.645679248Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.648703204Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.652193632Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.676753042Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.676833214Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.677013109Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.677022569Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.677180013Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.677187303Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.677344277Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.677351947Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.678035435Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.680996869Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.705100777Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.709283433Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.709307564Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.712386591Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.712450493Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.715454979Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.715673234Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.721508671Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.794519723Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.799084798Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.803817148Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.809022469Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.814584419Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.831122866Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.837701692Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:16.84315368Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.266782457Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.272790569Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.273199229Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.303102733Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.312029479Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.312754467Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.512361043Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.512883146Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.532215564Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.538691887Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.543203781Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.545632892Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.548109494Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.551802628Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.552542036Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.552552286Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.574280675Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.594151406Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.609577615Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.636308979Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.651288817Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.667298241Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 37.02819ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:31.840982133Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.84283364Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\") failed: Property not found. retry in 37.103271ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:31.857760676Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 65.549541ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:31.868893927Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 57.490295ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:31.88130945Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 26.956374ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:31.893128658Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\") failed: Property not found. retry in 26.230153ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:31.905022628Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 28.077645ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:31.916473337Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.917572455Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:31.917813341Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\") failed: Property not found. retry in 65.059078ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.421705423Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\") failed: Property not found. retry in 56.737187ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.456170192Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 49.54882ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.496204332Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 33.198064ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.531593435Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 102.556299ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.649752906Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 69.810699ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.739168641Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 61.157816ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.757270028Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 76.361755ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.778497354Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\") failed: Property not found. retry in 54.06388ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.802702274Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 64.21204ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.828252559Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:32.833988584Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\") failed: Property not found. retry in 170.290488ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.856745567Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\") failed: Property not found. retry in 142.94668ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.87744206Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 65.130656ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.893889544Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 110.408787ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.917699065Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 233.553327ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.934203172Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 263.982035ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.949063916Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_912c51fb-2723-4253-a072-17759e0be466:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:32.954459732Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 181.858223ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.971910163Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 250.183682ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:32.988415389Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\") failed: Property not found. retry in 223.274616ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.002844523Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 139.849915ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.017386Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 28.510262ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.031779163Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\") failed: Property not found. retry in 267.733805ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.045658113Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 26.758841ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.05742389Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\") failed: Property not found. retry in 273.729579ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.071144846Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 222.785948ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.084302908Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:33.092491835Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 132.042711ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.107637517Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:33.107729479Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 60.822798ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.130697499Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 138.434511ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.143220394Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:33.143288806Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 36.498629ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.163976568Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 596.698194ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.178126995Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 279.374085ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.19340046Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 124.529678ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.2092567Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 395.356122ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.22233167Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 349.035815ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.235564254Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\") failed: Property not found. retry in 337.910706ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.247790953Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 81.48778ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.257771544Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 257.579539ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.268476525Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 363.564345ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.281267047Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 212.418414ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.295034924Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 408.998961ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.321637816Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\") failed: Property not found. retry in 565.964694ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.336104371Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 241.970719ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.349903439Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\") failed: Property not found. retry in 474.404082ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.36501238Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 249.291611ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.380725466Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 57.942872ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.409523173Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 42.847509ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.422938141Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 147.654497ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.480188006Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 137.596375ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.491702166Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 610.07142ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.503305139Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 526.331169ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.523797296Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 623.811479ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.545533284Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 577.130342ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.604660776Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\") failed: Property not found. retry in 457.681822ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.623788518Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 730.94518ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.64287261Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 228.365091ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.657377676Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 1.151191841s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.676344104Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 185.760584ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.695763964Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 558.068561ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.712028354Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 493.119364ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.726881919Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 798.926782ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.747236203Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 532.023811ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.794024533Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\") failed: Property not found. retry in 958.015847ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.860328925Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 335.217961ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.898414566Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 262.974691ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.918317228Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\") failed: Property not found. retry in 968.726958ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:33.941073482Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 1.175658255s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.106241759Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\") failed: Property not found. retry in 725.703846ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.121538945Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 515.147378ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.136175344Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 1.016098507s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.189414377Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 1.18137592s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.205606936Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 1.449934482s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.224155214Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 838.526083ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.236338021Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 521.846944ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.249210626Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 900.869154ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.289498352Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 972.524802ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.34491244Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 945.658378ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.393449274Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 1.241504929s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.620609615Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:34.622677817Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:34.622895573Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 1.006084491s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.833513556Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 934.666059ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.852956117Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 838.784492ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.867532654Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:34.867829671Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\") failed: Property not found. retry in 1.441340992s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.884318707Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\") failed: Property not found. retry in 1.005077379s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.899172752Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:34.899358547Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:34.907212955Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:34.90741022Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\") failed: Property not found. retry in 675.798688ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:34.929493197Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 50.752853ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.020527193Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 56.137391ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.085279617Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 1.481847146s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.100636294Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 269.323618ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.158189776Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 734.716144ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.211373518Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 654.849127ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.229250339Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 63.727053ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.251055579Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 60.465487ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.312037298Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 1.407745316s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.330034492Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 1.108709576s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.34782474Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 140.400364ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.367276741Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 1.300021035s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.391445371Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 663.419931ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.430726132Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 570.678469ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.457567739Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 83.919245ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.474868075Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 260.378652ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.52498439Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 100.792151ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.575878554Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\") failed: Property not found. retry in 1.466650607s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.627316261Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 541.396174ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.694545237Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 391.930619ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.721550999Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 1.047904211s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.741310727Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 357.260394ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.809072947Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 1.06079623s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.840743065Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 1.122418946s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.869704486Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 748.708728ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.888830368Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 663.33033ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.909869659Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\") failed: Property not found. retry in 1.072747835s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.930945411Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 610.430225ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:35.967160054Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 663.511351ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.078460592Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 1.16304361s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.128257418Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 754.415721ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.165539919Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 994.0116ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.187958915Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 1.445216374s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.250889912Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\") failed: Property not found. retry in 778.337222ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.337792624Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 1.075269938s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.475011576Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 744.771314ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.595793603Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 1.376209713s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.621504822Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 665.701155ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.665540323Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 701.829827ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.705752157Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 1.352650666s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.740902624Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 886.162451ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.783901709Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 934.349125ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.811275409Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 544.053323ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.837494781Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 1.188935481s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.925356107Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 790.808789ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:36.946425579Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 1.465339374s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.014049515Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\") failed: Property not found. retry in 649.994072ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.034754787Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\") failed: Property not found. retry in 913.978424ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.123429744Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\") failed: Property not found. retry in 909.323972ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.14587812Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 800.221215ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.203561425Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 1.474866039s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.344331997Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 987.172581ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.363552792Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 549.82467ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.38172831Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 875.663592ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.40152083Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 683.826768ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.426454689Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 926.113921ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.569919778Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 1.040715836s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.686894469Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\") failed: Property not found. retry in 1.496627029s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.700112002Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 958.561484ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.7139042Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 1.037575672s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.751496149Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 1.016732171s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.763919662Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 1.033955088s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:37.942037945Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 1.441045314s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.01118545Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 603.565225ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.023613023Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\") failed: Property not found. retry in 1.295609261s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.050477691Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\") failed: Property not found. retry in 1.317048455s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.069008438Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 1.129723127s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.106819472Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 685.169399ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.123525054Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 999.270636ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.135879025Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 994.013831ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.292334272Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 1.143688447s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.365640192Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 1.188569288s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.492837301Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 678.049025ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.510917017Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 1.489321607s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.642360083Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 622.303065ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.68544923Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 1.338572095s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.741229967Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 1.079167098s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.794404528Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 624.053998ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.807894939Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 1.246699227s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.822118748Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 1.380780493s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.834221923Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 632.210513ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:38.989256833Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 1.299660771s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.147611918Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 909.038325ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.201549749Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\") failed: Property not found. retry in 1.341011223s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.212127926Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 534.084932ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.247059207Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 1.26549637s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.297609162Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 1.466047763s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.31890714Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\") failed: Property not found. retry in 1.476505278s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.358132259Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\") failed: Property not found. retry in 1.445544403s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.400675323Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 1.351704537s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.447546265Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 646.794716ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.467302423Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 1.495653285s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.523183553Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:39.526041675Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:39.526347443Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 1.036650965s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.635534988Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 1.041385052s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.696407993Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 74.286736ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.791999565Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 1.140309571s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:39.803374832Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 88.767646ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.109764601Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 878.67516ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.127378145Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 999.57854ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.139385238Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 943.818756ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.151160305Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 1.179653583s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.163206219Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 1.47960934s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.176209487Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 911.978207ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.188888127Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 277.372585ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.213394515Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 1.063064887s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.230170798Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 897.511738ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.465328751Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 525.010068ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.504418807Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\") failed: Property not found. retry in 1.021443305s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.566191946Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 1.31150266s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.581058001Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 1.315728974s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.68847511Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 1.113802856s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.754221079Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 919.228709ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.799940472Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 1.259951756s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.817027243Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b\") failed: Property not found. retry in 1.147108733s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.850952129Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6\") failed: Property not found. retry in 835.143779ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.86884169Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 769.346346ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:40.96040518Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 1.126750541s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.024491867Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808\") failed: Property not found. retry in 565.93322ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.037824713Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 1.187969477s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.054985856Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 658.785432ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.110408414Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0\") failed: Property not found. retry in 1.35113495s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.12607926Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 673.24299ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.153990594Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 618.223433ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.308636195Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 550.336433ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.358960005Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 508.217209ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.376037316Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b\") failed: Property not found. retry in 578.306401ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.603958775Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 1.176301099s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.67151887Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c\") failed: Property not found. retry in 867.790588ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.746361518Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 1.011955005s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.768414764Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 1.310952981s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.789590358Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb\") failed: Property not found. retry in 1.402655065s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.849703915Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b\") failed: Property not found. retry in 939.508422ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.889047027Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8\") failed: Property not found. retry in 910.052361ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.907897193Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 739.694274ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.929288402Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 645.27467ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.950900878Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 1.343006363s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:41.972633576Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 977.039215ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:42.044016607Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 919.398466ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:42.095578717Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 1.129009477s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:42.173487353Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 836.184045ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:42.263415511Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 684.742079ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:42.621065884Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 776.467184ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:42.693413929Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a\") failed: Property not found. retry in 1.222064152s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:42.805113447Z","hostname":"sock","pid":1590,"zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 852.76002ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:42.873500982Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 1.091054952s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:43.039675604Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 590.923754ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:43.06447199Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 944.955727ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:43.119277542Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 1.225881578s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:43.138547028Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77\") failed: Property not found. retry in 772.219608ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:43.323959686Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 738.932647ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:43.34670339Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f\") failed: Property not found. retry in 1.080347876s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:43.370150021Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 899.779205ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:43.490915458Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2\") failed: Property not found. retry in 1.005024638s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:43.674414507Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 1.122260095s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:43.744826903Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 923.623288ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:44.087032226Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1\") failed: Property not found. retry in 752.628188ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:44.109189865Z","hostname":"sock","pid":1590,"zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 1.190467637s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:44.15179276Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 1.071092697s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:44.384563602Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466\") failed: Property not found. retry in 1.343380909s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:44.411277586Z","hostname":"sock","pid":1590,"zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 1.293297654s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:44.914498061Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 597.619404ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:45.026316572Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:45.04050903Z","hostname":"sock","pid":1590,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"35.482798082s"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 721.366829ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:45.364243847Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 939.459593ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:45.472612501Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 1.477048075s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:45.642351013Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 608.391481ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:46.098369766Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 761.780004ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:46.218796504Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 1.386637109s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:46.42155794Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588\") failed: Property not found. retry in 1.225230581s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:46.716707395Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:46.900278597Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63321"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:46.942246805Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"ec505e3e-ce41-4c0a-878a-ccb75513b125","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63321","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":15260,"response_code":"200"} {"msg":"wait for service Some(\"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b\") failed: Property not found. retry in 1.206971773s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:46.990102152Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 1.13611334s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:47.129889239Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8\") failed: Property not found. retry in 1.18728262s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:47.819105726Z","hostname":"sock","pid":1590,"zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 544.713823ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:48.275606972Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e\") failed: Property not found. retry in 736.630044ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:22:48.829872035Z","hostname":"sock","pid":1590,"zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602034244Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_aa9efd65-76e0-40d3-b044-475ef541089b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602169828Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_66cf0e2c-6a1d-491a-b5ab-d43fdc957b77"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602186168Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_ed844809-41fb-4820-b590-15e558d9334b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602191388Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_2f2a8832-d4c0-4c7e-92b0-f06ffe750cfb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602196068Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_b369e86a-ba32-4e8e-81d4-099acec9e7b1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602200628Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_83e7a4a6-713d-420b-a2b9-eebccafbd808"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602205248Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_8c14321b-a1bb-4420-abee-713b270542b8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.60228467Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_055c12a1-5182-45fc-8015-ccd05d39636c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602292411Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_912c51fb-2723-4253-a072-17759e0be466"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602298921Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_f47e1510-c515-4500-92e8-33a6349000e8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602303451Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_70c53f9a-7915-490e-bd40-caf245b59e8f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602308091Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_420a9926-428f-4a96-ac2a-14aba90f476b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602312561Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_536bddb2-c38c-4e67-8b5d-9f1f1fb986e0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602321902Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_99d63837-89b8-441d-8f65-e85c20ba3ad6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602326442Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_6fa8b417-316d-4066-8027-98c8dac8136b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602330562Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_bb8e6fc4-c8a5-446a-bd73-00970284aef2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602348862Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_ba162515-e11a-42da-aa7b-aaf2fab53588"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602896846Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_oximeter_2584e6b7-8bd3-4591-b4dd-c3d456b7896a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.602902716Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_clickhouse_7409f646-a2ae-4503-8528-69a75169324e"} {"msg":"Writing ledger to /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.623808424Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.635228452Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.649504572Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"PUT","req_id":"1efec4d2-c31f-435b-9c7a-3d4b82f1b735","remote_addr":"[fd00:1122:3344:101::1]:64461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":34500257,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.791218257Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1212"} {"msg":"Writing ledger to /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.804085741Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.804490552Z","hostname":"sock","pid":1590,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.817309615Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:679"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.833553195Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:692"} {"msg":"rack_network_config: RackNetworkConfigV1 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV1 {\n addresses: [\n V4(\n Ipv4Network {\n addr: 10.151.2.174,\n prefix: 24,\n },\n ),\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Network {\n addr: 0.0.0.0,\n prefix: 0,\n },\n ),\n nexthop: 10.151.2.100,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Network {\n addr: fd00:1122:3344:100::,\n prefix: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:49.86983598Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:803"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:50.184632421Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:50.273875173Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:50.447257557Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:50.583624007Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:50.721026084Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:50.889355759Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:51.145711567Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:51.32899255Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:51.542285131Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:52.178726657Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:52.882332047Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:53.571126013Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:54.219796427Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:55.108776504Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:56.506275959Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:58.089002396Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/03a52263-53ab-4221-9c03-9637ab9d1295): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-04-30T22:22:59.170960781Z","hostname":"sock","pid":1590,"component":"ProbeManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/cbc2e9a2-a4c8-47b8-a5d7-5151055087a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:22:59.744529631Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:01.780948404Z","hostname":"sock","pid":1590,"component":"NexusNotifierTask","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.102420804Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:60476"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.107892322Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"308a2aa0-5437-452f-a8c3-511b76ec1eea","remote_addr":"[fd00:1122:3344:101::b]:60476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":5435,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.127436015Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:61310"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.139806537Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.228789932Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"d20febab-6f51-4626-8122-efb2c13e57d4","remote_addr":"[fd00:1122:3344:101::b]:61310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":101205,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.229047358Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:60571"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.229092579Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:34415"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.22910128Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40409"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.229193012Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63320"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.229242013Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.233028299Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.233225864Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.233316106Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f2ba5ec9-0f2d-4c1a-b4c3-d86b69934bd3","remote_addr":"[fd00:1122:3344:101::a]:40409","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4104,"response_code":"200"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.233355977Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.233387518Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"af452687-7b0f-4287-bfa9-516e81979ed3","remote_addr":"[fd00:1122:3344:101::c]:60571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3754,"response_code":"200"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.248498829Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.249777581Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"92e0f36d-8bf6-4565-924d-55d7b5da5bfa","remote_addr":"[fd00:1122:3344:101::b]:61310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":20471,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.327054961Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"fd46db86-d174-4f82-931e-5caf78a69521","remote_addr":"[fd00:1122:3344:101::c]:34415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":97849,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.327464601Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.327514932Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.32780111Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.328534678Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"32c483bf-9220-46e8-80a5-4804db0d3caf","remote_addr":"[fd00:1122:3344:101::c]:34415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1127,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.333117233Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"0c66e69a-4afc-47c8-859f-dedde981db06","remote_addr":"[fd00:1122:3344:101::a]:63320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":103831,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.335187746Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.335204696Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.335409741Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:02.336265973Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"dd8aa728-1841-46a8-9838-15df4bf0164b","remote_addr":"[fd00:1122:3344:101::a]:63320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1755,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7df41aaf-beea-4fdd-bc40-3ac2ab2c5b2e\", \"content-length\": \"124\", \"date\": \"Tue, 30 Apr 2024 22:23:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7df41aaf-beea-4fdd-bc40-3ac2ab2c5b2e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:03.004916581Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:855"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:03.077184354Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:52703"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:03.077309487Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:03.166635061Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"7b050967-22a1-43a7-b254-8163c42883e7","remote_addr":"[fd00:1122:3344:101::b]:52703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":89356,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:03.166941099Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:03.16699851Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:03.167302648Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:03.167563284Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"f8c6c47e-7c3d-4983-ae02-29b8df8a9ead","remote_addr":"[fd00:1122:3344:101::b]:52703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":702,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.275268363Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:36609"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.278545256Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 311f20b8-b719-44da-91c3-5352376ffda1, kind: Service { id: bb8e6fc4-c8a5-446a-bd73-00970284aef2 }, name: Name(\"external-dns-bb8e6fc4-c8a5-446a-bd73-00970284aef2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 251, 235])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 8eb857a7-b6e3-4ca0-9b74-1d3d6bb055b2, kind: Service { id: b369e86a-ba32-4e8e-81d4-099acec9e7b1 }, name: Name(\"external-dns-b369e86a-ba32-4e8e-81d4-099acec9e7b1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 159])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 05776b4c-39ae-42e4-9322-fa9f7f76e51b, kind: Service { id: f47e1510-c515-4500-92e8-33a6349000e8 }, name: Name(\"nexus-f47e1510-c515-4500-92e8-33a6349000e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 90])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 13a9087e-348f-469f-b05a-21b583fb04a2, kind: Service { id: ba162515-e11a-42da-aa7b-aaf2fab53588 }, name: Name(\"nexus-ba162515-e11a-42da-aa7b-aaf2fab53588\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 156, 33])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: af81916a-cbc4-462f-bb71-c50c92eb4656, kind: Service { id: 420a9926-428f-4a96-ac2a-14aba90f476b }, name: Name(\"nexus-420a9926-428f-4a96-ac2a-14aba90f476b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 158, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.283946742Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.284314171Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.284335612Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.284356212Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.284364732Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.284371192Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.284389143Z","hostname":"sock","pid":1590,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"152939ac-0476-43fa-a7ee-b97c8b328aff","remote_addr":"[fd00:1122:3344:101::c]:36609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8925,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.285317676Z","hostname":"sock","pid":1590,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:865"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.668165975Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:60872"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.668295598Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.749173498Z","hostname":"sock","pid":1590,"uri":"/inventory","method":"GET","req_id":"761368ae-20aa-4203-8999-a075b2a43835","remote_addr":"[fd00:1122:3344:101::c]:60872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":80934,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.749434905Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.749497527Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.749759163Z","hostname":"sock","pid":1590,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:06.750749818Z","hostname":"sock","pid":1590,"uri":"/omicron-zones","method":"GET","req_id":"e065083a-20fb-41b8-bada-1d137d18e344","remote_addr":"[fd00:1122:3344:101::c]:60872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1395,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:16.97833458Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62150"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:16.978503385Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"8127ded1-9b98-4ba6-87bd-6712563c688f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62150","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":64,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.175687861Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:59089"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.297543165Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"528c8dc7-63dd-4378-a50c-22a11733576c","remote_addr":"[fd00:1122:3344:101::b]:59089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":121799,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.299048363Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7742fe00-5e09-42b3-9a8d-ce6205e46857","remote_addr":"[fd00:1122:3344:101::b]:59089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1274,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.357566719Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.357693352Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.358958834Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.359958269Z","hostname":"sock","pid":1590,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.35998434Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"PUT","req_id":"8e8fdafa-34b7-49ea-bdf9-4fd0aac252ba","remote_addr":"[fd00:1122:3344:101::b]:59089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4179,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.567720431Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:33515"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.614437779Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"28791872-b183-4149-9dde-9b85b233d3a6","remote_addr":"[fd00:1122:3344:101::c]:33515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46649,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.614571632Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:45129"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.660764628Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"357b4922-52d4-40fe-a39c-bd9bfd162c0a","remote_addr":"[fd00:1122:3344:101::a]:45129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46124,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.660981603Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"94754860-094e-4efd-889d-9a7810c13073","remote_addr":"[fd00:1122:3344:101::c]:33515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":107,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:30.661195619Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"969f1b0e-4b90-4aea-be87-99c9f896bff9","remote_addr":"[fd00:1122:3344:101::a]:45129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:47.013489108Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58366"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:47.013570761Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"76d38f31-b1e8-4160-a9e5-a229080463ba","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58366","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":73,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:50.662545404Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:44689"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:23:50.672242408Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"7276a9f8-cef3-491f-bb98-007d83e44f40","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9286,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:00.422051586Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:33892"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:00.472298234Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"d2cb1828-9f92-40e2-9b64-a65dab8a966e","remote_addr":"[fd00:1122:3344:101::b]:33892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50189,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:00.472517277Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"6bb970fa-e82b-49b0-ab79-f1aa7e3d084b","remote_addr":"[fd00:1122:3344:101::b]:33892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:00.776644615Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:52186"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:00.827655058Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"eb8c88d7-43c6-474a-b4ad-fde57be9fbc5","remote_addr":"[fd00:1122:3344:101::c]:52186","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50855,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:00.828673803Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"cc5896bd-b86d-446f-beaa-fa990c4be594","remote_addr":"[fd00:1122:3344:101::c]:52186","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":849,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:00.951167783Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:47017"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:00.999951284Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"98e555b9-5c6a-46f6-85d4-4982a3aaeba7","remote_addr":"[fd00:1122:3344:101::a]:47017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48681,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:01.000746304Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"cf27cc0d-bd21-488e-bff4-4f3718e095ab","remote_addr":"[fd00:1122:3344:101::a]:47017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":639,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:17.047453607Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52398"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:17.047765145Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"f9c985c4-f40e-4573-b2a0-02f51bd101a4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52398","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":292,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:20.663290204Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"f91142d7-317c-4fdb-9faa-171c1f858977","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":543,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:30.134376913Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:60115"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:30.180361463Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"c4481bc6-5d7f-4b00-bb72-3f9c6d4b8547","remote_addr":"[fd00:1122:3344:101::b]:60115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45921,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:30.181071511Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"d01cb71f-ff5b-4244-ab30-c3914d9b0cd1","remote_addr":"[fd00:1122:3344:101::b]:60115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":582,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:30.555399054Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:35361"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:30.603453756Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"bf78b64d-353d-4e9a-b2e9-ef809a5f3457","remote_addr":"[fd00:1122:3344:101::c]:35361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47956,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:30.604305368Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"23237cca-b257-42c9-8d4d-90c97c92a5f6","remote_addr":"[fd00:1122:3344:101::c]:35361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":701,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:30.739859398Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:52487"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:30.78634472Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"1436a694-cf8e-4bf2-8067-521529cd1429","remote_addr":"[fd00:1122:3344:101::a]:52487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46407,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:30.786638828Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"13d311b1-c91d-4af0-979a-40775b3797bc","remote_addr":"[fd00:1122:3344:101::a]:52487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:47.081795077Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57326"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:47.082531745Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"53a56abe-324c-428c-982a-1ad46207ff04","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57326","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":723,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:24:50.662556019Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"a0f37292-fb7d-4617-8bcc-695598a2547d","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":673,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:00.271143036Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:49671"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:00.341135202Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"57b70c71-1cab-4538-9579-2365041517c7","remote_addr":"[fd00:1122:3344:101::b]:49671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":69923,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:00.341529752Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"16870549-7088-45ae-a7ff-ea8da6fe0d13","remote_addr":"[fd00:1122:3344:101::b]:49671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":156,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:00.719157688Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:61414"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:00.766152994Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"365eb035-32d4-4633-8b14-de5bdd655267","remote_addr":"[fd00:1122:3344:101::c]:61414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46933,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:00.76639322Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8d96addc-238c-4834-a150-5400f3886c68","remote_addr":"[fd00:1122:3344:101::c]:61414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:00.837161615Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60968"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:00.885842183Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e496a80a-eee6-4cb1-836d-da72c1399211","remote_addr":"[fd00:1122:3344:101::a]:60968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48566,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:00.886073659Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"d28232ef-6678-4cf1-a93e-1f99f58aefbd","remote_addr":"[fd00:1122:3344:101::a]:60968","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":89,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:02.076418008Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:17.115070619Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45793"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:17.115169551Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"9295cb57-efdc-43c4-b908-38c5542ee2d9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45793","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":82,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:20.662345099Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"275c27ae-5aad-4258-bd4f-b36a9ee0177c","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":944,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:30.078032705Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:53939"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:30.126927999Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"9e8a7341-3c7c-46ed-9147-271d56bfff45","remote_addr":"[fd00:1122:3344:101::b]:53939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48806,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:30.127707568Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f34344b2-1f05-403c-83dc-1dd171fae434","remote_addr":"[fd00:1122:3344:101::b]:53939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":569,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:30.560835925Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:43305"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:30.606416955Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"01d871ef-ac64-43fe-9a63-3835cedf4d93","remote_addr":"[fd00:1122:3344:101::a]:43305","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45517,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:30.606517687Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51471"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:30.610224511Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"0a18cfc6-14f6-4b67-894c-da7073731d1a","remote_addr":"[fd00:1122:3344:101::a]:43305","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3518,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:30.651476402Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"59550c5f-fb3b-484b-81ca-05c86b336ab7","remote_addr":"[fd00:1122:3344:101::c]:51471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44687,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:30.651887062Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"7681aeda-e191-4564-9022-6367bfa83a81","remote_addr":"[fd00:1122:3344:101::c]:51471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":166,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:47.147856919Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41717"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:47.147974602Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"21fa485c-b39b-4c26-921c-2b1756bf5831","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41717","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:25:50.680793245Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"7ce6001e-de4f-44df-bf42-ddcb2c180001","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":697,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:00.229170633Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:63539"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:00.280131369Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"1e9afad7-7496-4cb4-8d72-c3f6713205d4","remote_addr":"[fd00:1122:3344:101::b]:63539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50899,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:00.280476177Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"71f193ce-92c9-4e11-a394-570bf7b3dd9c","remote_addr":"[fd00:1122:3344:101::b]:63539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":137,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:00.684541311Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:56793"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:00.732154792Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"3bf0b760-5110-4bfc-88e0-0c7b13694723","remote_addr":"[fd00:1122:3344:101::c]:56793","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47512,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:00.732435009Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"11053dd6-4544-4461-9dfd-9fadc60cab68","remote_addr":"[fd00:1122:3344:101::c]:56793","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:00.751207543Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:54066"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:00.796623929Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6fdfff99-be30-4770-8b1f-f1fd0cf5e351","remote_addr":"[fd00:1122:3344:101::a]:54066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45297,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:00.796961087Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a4f81f16-1b1f-43cd-886c-769a3aac9156","remote_addr":"[fd00:1122:3344:101::a]:54066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:17.180315201Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35043"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:17.180484176Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"f1ee4d46-04b8-4afe-ae9f-eb96de91b4a4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35043","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":53,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:20.722137641Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"c23286fc-e862-40e1-a8ca-4307ff1d4509","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1068,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:30.113866056Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:35872"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:30.160328468Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"9cb988c7-b8fc-458f-b62e-39227697bcc5","remote_addr":"[fd00:1122:3344:101::b]:35872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46405,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:30.160621836Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"3238290d-c967-41f5-81b5-661a34939644","remote_addr":"[fd00:1122:3344:101::b]:35872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:30.557373615Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:42953"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:30.60197304Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"88e08e5b-4724-40fa-a83c-ae73b5c8c05b","remote_addr":"[fd00:1122:3344:101::a]:42953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44536,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:30.602192105Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"25388dde-cbc5-4b99-9241-46dd4ab67c9c","remote_addr":"[fd00:1122:3344:101::a]:42953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:30.648091723Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:53593"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:30.695648223Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"71993d09-8f9c-4a90-9ef9-e2e024e4c4df","remote_addr":"[fd00:1122:3344:101::c]:53593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47447,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:30.696522295Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"e435e0fe-fff0-487c-8ef9-42d2e2ee1a62","remote_addr":"[fd00:1122:3344:101::c]:53593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":615,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:47.212540874Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35976"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:47.213260013Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"9922dcbf-4847-43a9-a403-81342f1359a4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35976","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":697,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:26:50.722572572Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"896b2c73-f501-4b66-ad00-e25409701f02","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1202,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:00.334159694Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:48116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:00.38199069Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"34f5d3df-2980-4ede-a1ef-79f57c2a324b","remote_addr":"[fd00:1122:3344:101::b]:48116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47773,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:00.382238027Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"5475d67b-f126-4eab-aba4-2349042dd133","remote_addr":"[fd00:1122:3344:101::b]:48116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:00.700113836Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:63637"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:00.744535337Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"493b8302-f2c4-43e3-aef4-ecfd1ff6ccb0","remote_addr":"[fd00:1122:3344:101::c]:63637","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44310,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:00.744804603Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"1182942e-8ecb-4e95-b137-167f8bb820d1","remote_addr":"[fd00:1122:3344:101::c]:63637","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:00.804255653Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:59713"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:00.849378971Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"73f0a47e-9689-4e39-a1b9-3b5a29d3b337","remote_addr":"[fd00:1122:3344:101::a]:59713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45023,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:00.850242663Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"f0765187-c1d0-4a60-899f-d6698470c77f","remote_addr":"[fd00:1122:3344:101::a]:59713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":679,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:17.244717759Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57985"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:17.244822791Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"8d470606-fd99-4479-b644-3ffb04167b10","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57985","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":96,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:20.721998581Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"f8d5d8d9-2267-4563-912d-ae70422507e4","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":660,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:30.15899213Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:47183"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:30.204075558Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"27410751-0d53-4200-9f83-c7437119ce97","remote_addr":"[fd00:1122:3344:101::b]:47183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45045,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:30.204880268Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"64cd9034-c951-4278-9eaa-2afec4556d4b","remote_addr":"[fd00:1122:3344:101::b]:47183","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":660,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:30.589625234Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:49817"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:30.640474297Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"7f235b3b-d4d2-48af-927b-00b72a31da09","remote_addr":"[fd00:1122:3344:101::c]:49817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50728,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:30.640739374Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"99bd1c25-ab12-4c64-87e0-92b06a92fda9","remote_addr":"[fd00:1122:3344:101::c]:49817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:30.651609148Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:40391"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:30.70125387Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"da832dac-16b2-4a9a-b56f-5170fb6661c1","remote_addr":"[fd00:1122:3344:101::a]:40391","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49538,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:30.702004229Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"a62966f2-8b41-4e0c-a207-4475e95ba7b0","remote_addr":"[fd00:1122:3344:101::a]:40391","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":470,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:47.275973995Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64002"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:47.276091118Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"890e117b-a301-4806-b6bc-480193cb1f6c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64002","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":101,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:27:50.721106617Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"7ec1a122-f233-4600-97e9-206c8b0dc3ae","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":589,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:00.337547413Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:41836"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:00.388329754Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"a7b67835-1af6-4c3d-8f13-e3fa727d24e2","remote_addr":"[fd00:1122:3344:101::b]:41836","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50707,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:00.38857058Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b3e23a08-7840-4d02-ae21-95e7614000a4","remote_addr":"[fd00:1122:3344:101::b]:41836","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:00.667545378Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:58479"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:00.714894243Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"891975e9-dcb5-49fe-934b-ffdfa87fdc4d","remote_addr":"[fd00:1122:3344:101::a]:58479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47248,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:00.7151857Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"1ce118fb-f94f-4bb8-82aa-3f5eca1a0bd2","remote_addr":"[fd00:1122:3344:101::a]:58479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:00.771725286Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:36996"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:00.817687906Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"6f756e1c-9ae2-4507-9b9d-17477daf794b","remote_addr":"[fd00:1122:3344:101::c]:36996","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45879,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:00.818000154Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"dd51298b-1c93-4c32-91cc-ad9d3340e81c","remote_addr":"[fd00:1122:3344:101::c]:36996","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:17.309606541Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45334"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:17.310315179Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"f067ced5-6e44-4e7e-b8a6-c7dbec7b6cf5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45334","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":691,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:20.722024307Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"1c9a24bc-da36-44c8-8514-dd5ef8d4e48e","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:30.028988535Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:57122"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:30.076958345Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"1103e153-17f0-44b9-93ae-b18beda9d66f","remote_addr":"[fd00:1122:3344:101::b]:57122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47924,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:30.077241962Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"97dc853f-31b8-4f65-bc4b-6bb31ef96d9c","remote_addr":"[fd00:1122:3344:101::b]:57122","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:30.561227612Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:37580"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:30.608369001Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"5a0506dc-de70-4336-83cc-c60c5c092f3c","remote_addr":"[fd00:1122:3344:101::c]:37580","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47060,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:30.608602897Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"208555b2-2b84-4875-b235-72e7e669a6d0","remote_addr":"[fd00:1122:3344:101::c]:37580","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:30.616808754Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:44660"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:30.665523303Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"06690a0c-286c-4173-a4f4-69bded97db3c","remote_addr":"[fd00:1122:3344:101::a]:44660","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48596,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:30.666401325Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"71234a47-bb8d-4766-b4c4-0cfed28d9355","remote_addr":"[fd00:1122:3344:101::a]:44660","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":695,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:47.373714943Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60357"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:47.374229036Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"dbcf4441-542a-470c-91e1-91d4b00e47cf","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60357","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":61,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:28:50.807066147Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"b4e16cf9-2099-4f22-b385-99b31fba55a7","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":467,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:00.258244293Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:36751"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:00.309518657Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"08c5013d-14f7-4741-8c77-41c302bee602","remote_addr":"[fd00:1122:3344:101::b]:36751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":51211,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:00.309882046Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"bafd7fdc-4548-4303-bff3-a8d4ca38c8f2","remote_addr":"[fd00:1122:3344:101::b]:36751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:00.721751186Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:42320"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:00.771211824Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"523185ca-a6c3-47ef-abee-beeb02f55a2b","remote_addr":"[fd00:1122:3344:101::c]:42320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49387,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:00.772196599Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"92e4e7f9-c9d6-4924-8e6a-db36db0d0618","remote_addr":"[fd00:1122:3344:101::c]:42320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":833,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:00.946607668Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63822"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:00.99383613Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"54a6aaa8-73a8-4ed6-9c2f-d31fe253f4e7","remote_addr":"[fd00:1122:3344:101::a]:63822","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47147,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:00.994064496Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"b4a3c41b-e230-4ccd-882f-c6ce6bfc858c","remote_addr":"[fd00:1122:3344:101::a]:63822","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:17.40706506Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58191"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:17.407195793Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"bb7151eb-5593-4c24-a43c-0420a5ab4804","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58191","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":108,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:17.97592656Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:17.975999752Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:18.013523829Z","hostname":"sock","pid":1590,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999996789s","next_cleanup":"Instant { tv_sec: 5601, tv_nsec: 894682586 }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:20.806490788Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"50ed1d4b-053c-4750-998d-1908f6b3affd","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":468,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:30.052951299Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:58073"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:30.102255233Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"2e6475f2-c374-4769-a6fa-417924108e93","remote_addr":"[fd00:1122:3344:101::b]:58073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49262,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:30.10250784Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"757c4541-12c3-4ca8-9fbb-ec6b7e16a553","remote_addr":"[fd00:1122:3344:101::b]:58073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:30.498339995Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:52478"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:30.547024763Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"dbaa0ebb-071d-4951-8afa-bd3ed612686f","remote_addr":"[fd00:1122:3344:101::c]:52478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48616,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:30.547327581Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"738bfbac-3530-4054-bc27-18c09fa1a300","remote_addr":"[fd00:1122:3344:101::c]:52478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:30.663385629Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55475"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:30.712435727Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"e66c63e0-e2b1-437f-b919-a3616525caf6","remote_addr":"[fd00:1122:3344:101::a]:55475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48945,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:30.713408051Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"e884911b-27b2-4c76-9a62-daa5b819b804","remote_addr":"[fd00:1122:3344:101::a]:55475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":787,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:39.975985579Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:54199"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:39.987397167Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:521","metadata":"InstanceMetadata { silo_id: cf41c5e8-dd76-445f-baa7-2d8623d33b52, project_id: 90c76835-b152-4c7d-9e8f-a39c7ad7daed }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-04-30T22:29:39.490992Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(d55832e8-2225-411d-b99c-e29d444b1883), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-30T22:29:39.514946839Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 4326d05b-624b-44c0-bae4-f75ff4e0099c, kind: Instance { id: a0f714a7-798e-4802-b930-2ea484484d82 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 240, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7966715), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4326d05b-624b-44c0-bae4-f75ff4e0099c, kind: Instance { id: a0f714a7-798e-4802-b930-2ea484484d82 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 240, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7966715), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4326d05b-624b-44c0-bae4-f75ff4e0099c, kind: Instance { id: a0f714a7-798e-4802-b930-2ea484484d82 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 240, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7966715), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(7966715))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4326d05b-624b-44c0-bae4-f75ff4e0099c, kind: Instance { id: a0f714a7-798e-4802-b930-2ea484484d82 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 240, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7966715), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-99e49e360c3\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 8f44b8f0-e4ca-4540-ba55-905a399ebddd, read_only_parent: Some(Volume { block_size: 512, id: 778bf4b7-3436-4a30-90e4-df951fdb71dc, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 445d3a40-ff99-4193-a2bc-4099e2995848, key: Some(\"GEInoBbkrrf4bOx0/RHUkVXrjzUZ9/ExzjT0ywjMhyM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::17]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 8f44b8f0-e4ca-4540-ba55-905a399ebddd, key: Some(\"i/yrG2L45FUFQyn0m1xw5UmVCB1CXSghEXDDmCnD3ZA=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19000\", \"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAADBs7OeWJ5YAACzs55YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAwrOznlieWAAAs7OeWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJhMGY3MTRhNy03OThlLTQ4MDItYjkzMC0yZWE0ODQ0ODRkODIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJR1d5WTBXcU1ETGZMS3JpSi9kdjZZSk4wY3lPTXBYSkdvSW1VRUNmUlJyWCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"d55832e8-2225-411d-b99c-e29d444b1883","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:39.98750326Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:39.987845818Z","hostname":"sock","pid":1590,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 4326d05b-624b-44c0-bae4-f75ff4e0099c, kind: Instance { id: a0f714a7-798e-4802-b930-2ea484484d82 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 240, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7966715), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4326d05b-624b-44c0-bae4-f75ff4e0099c, kind: Instance { id: a0f714a7-798e-4802-b930-2ea484484d82 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 240, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7966715), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4326d05b-624b-44c0-bae4-f75ff4e0099c, kind: Instance { id: a0f714a7-798e-4802-b930-2ea484484d82 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 240, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7966715), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(7966715))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 4326d05b-624b-44c0-bae4-f75ff4e0099c, kind: Instance { id: a0f714a7-798e-4802-b930-2ea484484d82 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 250, 240, 198])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(7966715), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-99e49e360c3\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 8f44b8f0-e4ca-4540-ba55-905a399ebddd, read_only_parent: Some(Volume { block_size: 512, id: 778bf4b7-3436-4a30-90e4-df951fdb71dc, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 445d3a40-ff99-4193-a2bc-4099e2995848, key: Some(\"GEInoBbkrrf4bOx0/RHUkVXrjzUZ9/ExzjT0ywjMhyM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::17]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 8f44b8f0-e4ca-4540-ba55-905a399ebddd, key: Some(\"i/yrG2L45FUFQyn0m1xw5UmVCB1CXSghEXDDmCnD3ZA=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19000\", \"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAADBs7OeWJ5YAACzs55YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAwrOznlieWAAAs7OeWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJhMGY3MTRhNy03OThlLTQ4MDItYjkzMC0yZWE0ODQ0ODRkODIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJR1d5WTBXcU1ETGZMS3JpSi9kdjZZSk4wY3lPTXBYSkdvSW1VRUNmUlJyWCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(d55832e8-2225-411d-b99c-e29d444b1883), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-30T22:29:39.514946839Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-04-30T22:29:39.490992Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"d55832e8-2225-411d-b99c-e29d444b1883","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:39.987937931Z","hostname":"sock","pid":1590,"uri":"/instances/a0f714a7-798e-4802-b930-2ea484484d82","method":"PUT","req_id":"cb85c727-a956-4cac-a1c8-40401e1304fb","remote_addr":"[fd00:1122:3344:101::a]:54199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":11884,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:40.123014748Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:63013"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:40.124029694Z","hostname":"sock","pid":1590,"uri":"/instances/a0f714a7-798e-4802-b930-2ea484484d82/state","method":"PUT","req_id":"ffadbeb1-bb4d-4981-acac-833d580ecd52","remote_addr":"[fd00:1122:3344:101::a]:63013","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":954,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:40.146333646Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 250, 240, 198]), slot: 0, vni: Vni { inner: 7966715 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:40.146414809Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:40.180866088Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:40.202938805Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:42.413152362Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:42.413385208Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883\") failed: Property not found. retry in 62.446589ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:29:42.597093413Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883\") failed: Property not found. retry in 143.075658ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:29:42.681129553Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883\") failed: Property not found. retry in 251.625059ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:29:42.844626237Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883\") failed: Property not found. retry in 296.320252ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:29:43.106911293Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883\") failed: Property not found. retry in 555.999809ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:29:43.412568924Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883\") failed: Property not found. retry in 899.531887ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:29:43.980099262Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883\") failed: Property not found. retry in 1.08510452s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:29:44.88976508Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883\") failed: Property not found. retry in 765.41824ms","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:29:45.984487237Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883\") failed: Property not found. retry in 1.01180875s","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:29:46.760106244Z","hostname":"sock","pid":1590,"zone":"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:47.438794445Z","hostname":"sock","pid":1590,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35152"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:47.43899281Z","hostname":"sock","pid":1590,"uri":"/baseboard","method":"GET","req_id":"2ec97565-dc88-468f-af1f-673f6d3c829c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35152","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":116,"response_code":"200"} {"msg":"Started propolis in zone: oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:47.801560157Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:47.810361559Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:47.847199008Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAADBs7OeWJ5YAACzs55YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAwrOznlieWAAAs7OeWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJhMGY3MTRhNy03OThlLTQ4MDItYjkzMC0yZWE0ODQ0ODRkODIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJR1d5WTBXcU1ETGZMS3JpSi9kdjZZSk4wY3lPTXBYSkdvSW1VRUNmUlJyWCJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-99e49e360c3\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 8f44b8f0-e4ca-4540-ba55-905a399ebddd, read_only_parent: Some(Volume { block_size: 512, id: 778bf4b7-3436-4a30-90e4-df951fdb71dc, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 445d3a40-ff99-4193-a2bc-4099e2995848, key: Some(\"GEInoBbkrrf4bOx0/RHUkVXrjzUZ9/ExzjT0ywjMhyM=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::17]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 8f44b8f0-e4ca-4540-ba55-905a399ebddd, key: Some(\"i/yrG2L45FUFQyn0m1xw5UmVCB1CXSghEXDDmCnD3ZA=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19000\", \"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: a0f714a7-798e-4802-b930-2ea484484d82, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 90c76835-b152-4c7d-9e8f-a39c7ad7daed, silo_id: cf41c5e8-dd76-445f-baa7-2d8623d33b52 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:47.847812433Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:47.932200052Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-04-30T22:29:47.933424813Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:47.933440743Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:47.933450393Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-04-30T22:29:47.933424813Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(d55832e8-2225-411d-b99c-e29d444b1883), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-30T22:29:39.514946839Z }","propolis_id":"d55832e8-2225-411d-b99c-e29d444b1883"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:47.933466274Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d55832e8-2225-411d-b99c-e29d444b1883), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-30T22:29:39.514946839Z }, propolis_id: d55832e8-2225-411d-b99c-e29d444b1883, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-04-30T22:29:47.933424813Z } }","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-04-30T22:29:48.005983493Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:48.005999563Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:48.006016804Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-04-30T22:29:48.005983493Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(d55832e8-2225-411d-b99c-e29d444b1883), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-30T22:29:39.514946839Z }","propolis_id":"d55832e8-2225-411d-b99c-e29d444b1883"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:48.006026804Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d55832e8-2225-411d-b99c-e29d444b1883), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-30T22:29:39.514946839Z }, propolis_id: d55832e8-2225-411d-b99c-e29d444b1883, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-04-30T22:29:48.005983493Z } }","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:50.80718016Z","hostname":"sock","pid":1590,"uri":"//03a52263-53ab-4221-9c03-9637ab9d1295","method":"GET","req_id":"a09261db-8d93-4290-bd63-5fda1dc1e278","remote_addr":"[fd00:1122:3344:101::d]:44689","local_addr":"[fd00:1122:3344:101::1]:35283","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","component":"producer-server","component":"MetricsManager","sled_id":"03a52263-53ab-4221-9c03-9637ab9d1295","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":760,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:56.465531084Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:42896"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:56.466909559Z","hostname":"sock","pid":1590,"uri":"/instances/a0f714a7-798e-4802-b930-2ea484484d82/state","method":"PUT","req_id":"d8032309-1aa3-42a2-bce2-4673fc38b91c","remote_addr":"[fd00:1122:3344:101::a]:42896","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":523,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-04-30T22:29:56.467022591Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:56.467047022Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:56.467070563Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-04-30T22:29:56.467022591Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(d55832e8-2225-411d-b99c-e29d444b1883), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-30T22:29:39.514946839Z }","propolis_id":"d55832e8-2225-411d-b99c-e29d444b1883"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:56.467101373Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(d55832e8-2225-411d-b99c-e29d444b1883), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-04-30T22:29:39.514946839Z }, propolis_id: d55832e8-2225-411d-b99c-e29d444b1883, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-04-30T22:29:56.467022591Z } }","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-04-30T22:29:56.507958944Z }","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:56.507975995Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:56.508015665Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-04-30T22:29:56.507958944Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-30T22:29:56.507958944Z }","propolis_id":"d55832e8-2225-411d-b99c-e29d444b1883"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:56.508040656Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:56.508046376Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:56.508226371Z","hostname":"sock","pid":1590,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:56.508275252Z","hostname":"sock","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/8a27d52c-0cf2-44eb-a59a-d16436f25542/debug/bundle/zone\", \"/pool/int/c953fe1d-1619-477a-bac7-ceb27e5a468a/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/9a1451e3-b124-44f6-8078-fbc416821fc9/crypt/debug\", \"/pool/ext/52a73ea1-1580-48fa-9393-5bb705994a44/crypt/debug\", \"/pool/ext/03d465fc-bc73-4466-af31-1a5670be772e/crypt/debug\", \"/pool/ext/145ad0b0-5e5c-4755-ba6c-da149e549070/crypt/debug\", \"/pool/ext/b110b23f-2145-46b8-8de0-dc91bc41dd55/crypt/debug\", \"/pool/ext/d3fbf4c0-71ec-480f-942f-6fd5fd537b7c/crypt/debug\", \"/pool/ext/c8e411fe-43b9-47e1-88c5-1303772b3e4e/crypt/debug\", \"/pool/ext/e10de419-fa2f-479e-b380-a88997c87dc2/crypt/debug\", \"/pool/ext/bb74811e-acf4-486b-913b-bc09ce8f6984/crypt/debug\"] }","zone_name":"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:57.150545134Z","hostname":"sock","pid":1590,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883\", bundle_id: 37be7205-46a0-4da1-9649-8cfb9a946fa4 }, time_created: 2024-04-30T22:29:56.509022971Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_d55832e8-2225-411d-b99c-e29d444b1883","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:29:57.150625367Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:58.868905334Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:58.893833233Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-30T22:29:56.507958944Z }, propolis_id: d55832e8-2225-411d-b99c-e29d444b1883, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-04-30T22:29:58.893763971Z } }","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:58.995328413Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:64307"} {"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-04-30T22:29:58.99675854Z","hostname":"sock","pid":1590,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:58.997936179Z","hostname":"sock","pid":1590,"uri":"/v2p/4326d05b-624b-44c0-bae4-f75ff4e0099c","method":"DELETE","req_id":"296b3ae6-5813-48c2-9dab-efdf48916564","remote_addr":"[fd00:1122:3344:101::b]:64307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2512,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:59.28851517Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-04-30T22:29:56.507958944Z }, propolis_id: d55832e8-2225-411d-b99c-e29d444b1883, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-04-30T22:29:58.893763971Z } }","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:29:59.292377257Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"} {"msg":"Failed to publish state to Nexus, will not retry: Notification(Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3cc4ec3d-d8b6-4913-a295-7442663b7a96\", \"content-length\": \"177\", \"date\": \"Tue, 30 Apr 2024 22:29:58 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: instance with id \\\"a0f714a7-798e-4802-b930-2ea484484d82\\\"\", request_id: \"3cc4ec3d-d8b6-4913-a295-7442663b7a96\" })","v":0,"name":"SledAgent","level":50,"time":"2024-04-30T22:29:59.34758095Z","hostname":"sock","pid":1590,"instance_id":"a0f714a7-798e-4802-b930-2ea484484d82","component":"InstanceManager","file":"sled-agent/src/instance.rs:555","instance_id":"a0f714a7-798e-4802-b930-2ea484484d82"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:30:00.207892983Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:46245"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:30:00.264299086Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"79fbb329-c3f6-4b28-949e-8eead09e689f","remote_addr":"[fd00:1122:3344:101::b]:46245","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":56352,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:30:00.264638125Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"8fe77998-bc13-43ae-9938-21c2d0800f29","remote_addr":"[fd00:1122:3344:101::b]:46245","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:30:00.754048521Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:57106"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:30:00.836531752Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"d654ea31-9285-4dbc-81a4-12288caca578","remote_addr":"[fd00:1122:3344:101::a]:57106","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":82380,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:30:00.836795949Z","hostname":"sock","pid":1590,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:49098"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:30:00.837073696Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"d4d46b75-8cab-4017-a626-8b01bf21e39f","remote_addr":"[fd00:1122:3344:101::a]:57106","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":237,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:30:00.899791337Z","hostname":"sock","pid":1590,"uri":"/switch-ports","method":"POST","req_id":"ea738cb5-40a8-49c4-90c4-27a77df2acac","remote_addr":"[fd00:1122:3344:101::c]:49098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":62864,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:30:00.900192307Z","hostname":"sock","pid":1590,"uri":"/network-bootstore-config","method":"GET","req_id":"1634d363-d178-43f6-b461-3344df61e1c4","remote_addr":"[fd00:1122:3344:101::c]:49098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":128,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-04-30T22:30:02.076703488Z","hostname":"sock","pid":1590,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}